From 6dcd624b80b33b5c1bf25b13ba660aa1b6d287eb Mon Sep 17 00:00:00 2001 From: Cedric CAFFY <cedric.caffy@cern.ch> Date: Tue, 4 Feb 2020 10:56:47 +0100 Subject: [PATCH] Added Statistics Schema + CppSchemaStatementsReader to compare the statistics database against its schema --- catalogue/CMakeLists.txt | 2 +- ...aGetter.cpp => DatabaseMetadataGetter.cpp} | 72 +++++++++---------- ...aGetter.hpp => DatabaseMetadataGetter.hpp} | 48 ++++++------- catalogue/SQLiteSchemaComparer.cpp | 50 +++++++------ catalogue/SQLiteSchemaComparer.hpp | 9 +-- catalogue/SchemaChecker.cpp | 69 +++++++++++++----- catalogue/SchemaChecker.hpp | 24 +++++-- catalogue/SchemaComparer.cpp | 9 +-- catalogue/SchemaComparer.hpp | 14 ++-- catalogue/SchemaSqlStatementsReader.cpp | 11 +-- catalogue/SchemaSqlStatementsReader.hpp | 9 ++- catalogue/VerifySchemaCmd.cpp | 4 +- statistics/CMakeLists.txt | 32 +++++++-- .../MysqlStatisticsSchema.before_SQL.cpp | 33 +++++++++ statistics/MysqlStatisticsSchema.hpp | 37 ++++++++++ statistics/StatisticsCmd.cpp | 33 ++++++--- statistics/StatisticsSchema.cpp | 24 +++++++ statistics/StatisticsSchema.hpp | 51 +++++++++++++ statistics/common_statistics_schema.sql | 5 ++ 19 files changed, 388 insertions(+), 148 deletions(-) rename catalogue/{CatalogueMetadataGetter.cpp => DatabaseMetadataGetter.cpp} (71%) rename catalogue/{CatalogueMetadataGetter.hpp => DatabaseMetadataGetter.hpp} (73%) create mode 100644 statistics/MysqlStatisticsSchema.before_SQL.cpp create mode 100644 statistics/MysqlStatisticsSchema.hpp create mode 100644 statistics/StatisticsSchema.cpp create mode 100644 statistics/StatisticsSchema.hpp create mode 100644 statistics/common_statistics_schema.sql diff --git a/catalogue/CMakeLists.txt b/catalogue/CMakeLists.txt index 186e23f91d..b1d8591ff0 100644 --- a/catalogue/CMakeLists.txt +++ b/catalogue/CMakeLists.txt @@ -297,7 +297,7 @@ set (SCHEMA_CHECKER_LIB_SRC_FILES DbToSQLiteStatementTransformer.cpp SchemaComparerResult.cpp SchemaChecker.cpp - CatalogueMetadataGetter.cpp + DatabaseMetadataGetter.cpp CatalogueSchema.cpp OracleCatalogueSchema.cpp SqliteCatalogueSchema.cpp diff --git a/catalogue/CatalogueMetadataGetter.cpp b/catalogue/DatabaseMetadataGetter.cpp similarity index 71% rename from catalogue/CatalogueMetadataGetter.cpp rename to catalogue/DatabaseMetadataGetter.cpp index 6c4cd9dc2a..15ac56799e 100644 --- a/catalogue/CatalogueMetadataGetter.cpp +++ b/catalogue/DatabaseMetadataGetter.cpp @@ -16,7 +16,7 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ -#include "CatalogueMetadataGetter.hpp" +#include "DatabaseMetadataGetter.hpp" #include <algorithm> namespace cta { @@ -53,9 +53,9 @@ void MetadataGetter::removeObjectNameMatches(std::list<std::string> &objects, co }); } -CatalogueMetadataGetter::CatalogueMetadataGetter(cta::rdbms::Conn& conn):m_conn(conn){} +DatabaseMetadataGetter::DatabaseMetadataGetter(cta::rdbms::Conn& conn):m_conn(conn){} -SchemaVersion CatalogueMetadataGetter::getCatalogueVersion(){ +SchemaVersion DatabaseMetadataGetter::getCatalogueVersion(){ const char *const sql = "SELECT " "CTA_CATALOGUE.SCHEMA_VERSION_MAJOR AS SCHEMA_VERSION_MAJOR," @@ -104,13 +104,13 @@ SchemaVersion CatalogueMetadataGetter::getCatalogueVersion(){ } } -std::list<std::string> CatalogueMetadataGetter::getTableNames(){ +std::list<std::string> DatabaseMetadataGetter::getTableNames(){ std::list<std::string> tableNames = m_conn.getTableNames(); removeObjectNameContaining(tableNames,{"DATABASECHANGELOG","DATABASECHANGELOGLOCK"}); return tableNames; } -std::list<std::string> CatalogueMetadataGetter::getIndexNames(){ +std::list<std::string> DatabaseMetadataGetter::getIndexNames(){ std::list<std::string> indexNames = m_conn.getIndexNames(); //We just want indexes created by the user, their name are finishing by _IDX or by _I cta::utils::Regex regexIndexes("(.*_IDX$)|(.*_I$)"); @@ -118,74 +118,74 @@ std::list<std::string> CatalogueMetadataGetter::getIndexNames(){ return indexNames; } -std::map<std::string,std::string> CatalogueMetadataGetter::getColumns(const std::string& tableName){ +std::map<std::string,std::string> DatabaseMetadataGetter::getColumns(const std::string& tableName){ return m_conn.getColumns(tableName); } -std::list<std::string> CatalogueMetadataGetter::getConstraintNames(const std::string &tableName){ +std::list<std::string> DatabaseMetadataGetter::getConstraintNames(const std::string &tableName){ std::list<std::string> constraintNames = m_conn.getConstraintNames(tableName); //This constraint is added by ALTER TABLE, we can't check its existence for now removeObjectNameContaining(constraintNames,{"CATALOGUE_STATUS_CONTENT_CK"}); return constraintNames; } -std::list<std::string> CatalogueMetadataGetter::getParallelTableNames(){ +std::list<std::string> DatabaseMetadataGetter::getParallelTableNames(){ return m_conn.getParallelTableNames(); } -CatalogueMetadataGetter::~CatalogueMetadataGetter() {} +DatabaseMetadataGetter::~DatabaseMetadataGetter() {} -SQLiteCatalogueMetadataGetter::SQLiteCatalogueMetadataGetter(cta::rdbms::Conn & conn):CatalogueMetadataGetter(conn){} -SQLiteCatalogueMetadataGetter::~SQLiteCatalogueMetadataGetter(){} +SQLiteDatabaseMetadataGetter::SQLiteDatabaseMetadataGetter(cta::rdbms::Conn & conn):DatabaseMetadataGetter(conn){} +SQLiteDatabaseMetadataGetter::~SQLiteDatabaseMetadataGetter(){} -std::list<std::string> SQLiteCatalogueMetadataGetter::getIndexNames() { - std::list<std::string> indexNames = CatalogueMetadataGetter::getIndexNames(); +std::list<std::string> SQLiteDatabaseMetadataGetter::getIndexNames() { + std::list<std::string> indexNames = DatabaseMetadataGetter::getIndexNames(); //We do not want the sqlite_autoindex created automatically by SQLite removeObjectNameContaining(indexNames,{"sqlite_autoindex"}); return indexNames; } -std::list<std::string> SQLiteCatalogueMetadataGetter::getTableNames(){ - std::list<std::string> tableNames = CatalogueMetadataGetter::getTableNames(); +std::list<std::string> SQLiteDatabaseMetadataGetter::getTableNames(){ + std::list<std::string> tableNames = DatabaseMetadataGetter::getTableNames(); //We do not want the sqlite_sequence tables created automatically by SQLite removeObjectNameContaining(tableNames,{"sqlite_sequence"}); return tableNames; } -cta::rdbms::Login::DbType SQLiteCatalogueMetadataGetter::getDbType(){ +cta::rdbms::Login::DbType SQLiteDatabaseMetadataGetter::getDbType(){ return cta::rdbms::Login::DbType::DBTYPE_SQLITE; } -OracleCatalogueMetadataGetter::OracleCatalogueMetadataGetter(cta::rdbms::Conn & conn):CatalogueMetadataGetter(conn){} -OracleCatalogueMetadataGetter::~OracleCatalogueMetadataGetter(){} -cta::rdbms::Login::DbType OracleCatalogueMetadataGetter::getDbType(){ +OracleDatabaseMetadataGetter::OracleDatabaseMetadataGetter(cta::rdbms::Conn & conn):DatabaseMetadataGetter(conn){} +OracleDatabaseMetadataGetter::~OracleDatabaseMetadataGetter(){} +cta::rdbms::Login::DbType OracleDatabaseMetadataGetter::getDbType(){ return cta::rdbms::Login::DbType::DBTYPE_ORACLE; } -MySQLCatalogueMetadataGetter::MySQLCatalogueMetadataGetter(cta::rdbms::Conn& conn):CatalogueMetadataGetter(conn) {} -MySQLCatalogueMetadataGetter::~MySQLCatalogueMetadataGetter(){} -cta::rdbms::Login::DbType MySQLCatalogueMetadataGetter::getDbType(){ +MySQLDatabaseMetadataGetter::MySQLDatabaseMetadataGetter(cta::rdbms::Conn& conn):DatabaseMetadataGetter(conn) {} +MySQLDatabaseMetadataGetter::~MySQLDatabaseMetadataGetter(){} +cta::rdbms::Login::DbType MySQLDatabaseMetadataGetter::getDbType(){ return cta::rdbms::Login::DbType::DBTYPE_MYSQL; } -PostgresCatalogueMetadataGetter::PostgresCatalogueMetadataGetter(cta::rdbms::Conn& conn):CatalogueMetadataGetter(conn) {} -PostgresCatalogueMetadataGetter::~PostgresCatalogueMetadataGetter(){} -cta::rdbms::Login::DbType PostgresCatalogueMetadataGetter::getDbType(){ +PostgresDatabaseMetadataGetter::PostgresDatabaseMetadataGetter(cta::rdbms::Conn& conn):DatabaseMetadataGetter(conn) {} +PostgresDatabaseMetadataGetter::~PostgresDatabaseMetadataGetter(){} +cta::rdbms::Login::DbType PostgresDatabaseMetadataGetter::getDbType(){ return cta::rdbms::Login::DbType::DBTYPE_POSTGRESQL; } -CatalogueMetadataGetter * CatalogueMetadataGetterFactory::create(const rdbms::Login::DbType dbType, cta::rdbms::Conn & conn) { +DatabaseMetadataGetter * DatabaseMetadataGetterFactory::create(const rdbms::Login::DbType dbType, cta::rdbms::Conn & conn) { typedef rdbms::Login::DbType DbType; switch(dbType){ case DbType::DBTYPE_IN_MEMORY: case DbType::DBTYPE_SQLITE: - return new SQLiteCatalogueMetadataGetter(conn); + return new SQLiteDatabaseMetadataGetter(conn); case DbType::DBTYPE_ORACLE: - return new OracleCatalogueMetadataGetter(conn); + return new OracleDatabaseMetadataGetter(conn); case DbType::DBTYPE_MYSQL: - return new MySQLCatalogueMetadataGetter(conn); + return new MySQLDatabaseMetadataGetter(conn); case DbType::DBTYPE_POSTGRESQL: - return new PostgresCatalogueMetadataGetter(conn); + return new PostgresDatabaseMetadataGetter(conn); default: throw cta::exception::Exception("In CatalogueMetadataGetterFactory::create(), can't get CatalogueMetadataGetter for dbType "+rdbms::Login::dbTypeToString(dbType)); } @@ -194,24 +194,24 @@ CatalogueMetadataGetter * CatalogueMetadataGetterFactory::create(const rdbms::Lo /** * SCHEMA METADATA GETTER methods */ -SchemaMetadataGetter::SchemaMetadataGetter(std::unique_ptr<SQLiteCatalogueMetadataGetter> sqliteCatalogueMetadataGetter, const cta::rdbms::Login::DbType dbType):m_dbType(dbType) { - m_sqliteCatalogueMetadataGetter = std::move(sqliteCatalogueMetadataGetter); +SchemaMetadataGetter::SchemaMetadataGetter(std::unique_ptr<SQLiteDatabaseMetadataGetter> sqliteCatalogueMetadataGetter, const cta::rdbms::Login::DbType dbType):m_dbType(dbType) { + m_sqliteDatabaseMetadataGetter = std::move(sqliteCatalogueMetadataGetter); } std::list<std::string> SchemaMetadataGetter::getIndexNames() { - return m_sqliteCatalogueMetadataGetter->getIndexNames(); + return m_sqliteDatabaseMetadataGetter->getIndexNames(); } std::list<std::string> SchemaMetadataGetter::getTableNames() { - return m_sqliteCatalogueMetadataGetter->getTableNames(); + return m_sqliteDatabaseMetadataGetter->getTableNames(); } std::map<std::string,std::string> SchemaMetadataGetter::getColumns(const std::string& tableName) { - return m_sqliteCatalogueMetadataGetter->getColumns(tableName); + return m_sqliteDatabaseMetadataGetter->getColumns(tableName); } std::list<std::string> SchemaMetadataGetter::getConstraintNames(const std::string& tableName) { - std::list<std::string> constraintNames = m_sqliteCatalogueMetadataGetter->getConstraintNames(tableName); + std::list<std::string> constraintNames = m_sqliteDatabaseMetadataGetter->getConstraintNames(tableName); if(m_dbType == cta::rdbms::Login::DbType::DBTYPE_POSTGRESQL){ //If the database to compare is POSTGRESQL, we cannot compare NOT NULL CONSTRAINT names //indeed, POSTGRESQL can not give the NOT NULL constraint names diff --git a/catalogue/CatalogueMetadataGetter.hpp b/catalogue/DatabaseMetadataGetter.hpp similarity index 73% rename from catalogue/CatalogueMetadataGetter.hpp rename to catalogue/DatabaseMetadataGetter.hpp index cadabe86dc..d28c010cd5 100644 --- a/catalogue/CatalogueMetadataGetter.hpp +++ b/catalogue/DatabaseMetadataGetter.hpp @@ -49,12 +49,12 @@ public: * This class is used to get the Metadata (table names, columns, indexes) of the database accessed via the connection given in parameters * It will simply call the methods from the connection (Conn) instance and adapt or not the metadata returned. */ -class CatalogueMetadataGetter: public MetadataGetter { +class DatabaseMetadataGetter: public MetadataGetter { protected: rdbms::Conn& m_conn; public: - CatalogueMetadataGetter(cta::rdbms::Conn & conn); - virtual ~CatalogueMetadataGetter(); + DatabaseMetadataGetter(cta::rdbms::Conn & conn); + virtual ~DatabaseMetadataGetter(); SchemaVersion getCatalogueVersion(); virtual std::list<std::string> getIndexNames(); virtual std::list<std::string> getTableNames(); @@ -65,56 +65,56 @@ class CatalogueMetadataGetter: public MetadataGetter { }; /** - * Specific SQLite Catalogue metadata getter + * Specific SQLite database metadata getter */ -class SQLiteCatalogueMetadataGetter: public CatalogueMetadataGetter{ +class SQLiteDatabaseMetadataGetter: public DatabaseMetadataGetter{ public: - SQLiteCatalogueMetadataGetter(cta::rdbms::Conn & conn); - virtual ~SQLiteCatalogueMetadataGetter(); + SQLiteDatabaseMetadataGetter(cta::rdbms::Conn & conn); + virtual ~SQLiteDatabaseMetadataGetter(); std::list<std::string> getIndexNames() override; std::list<std::string> getTableNames() override; cta::rdbms::Login::DbType getDbType() override; }; /** - * Specific Oracle Catalogue metadata getter + * Specific Oracle database metadata getter */ -class OracleCatalogueMetadataGetter: public CatalogueMetadataGetter{ +class OracleDatabaseMetadataGetter: public DatabaseMetadataGetter{ public: - OracleCatalogueMetadataGetter(cta::rdbms::Conn & conn); + OracleDatabaseMetadataGetter(cta::rdbms::Conn & conn); cta::rdbms::Login::DbType getDbType() override; - virtual ~OracleCatalogueMetadataGetter(); + virtual ~OracleDatabaseMetadataGetter(); }; /** - * Specific MySQL Catalogue metadata getter + * Specific MySQL database metadata getter */ -class MySQLCatalogueMetadataGetter: public CatalogueMetadataGetter{ +class MySQLDatabaseMetadataGetter: public DatabaseMetadataGetter{ public: - MySQLCatalogueMetadataGetter(cta::rdbms::Conn &conn); + MySQLDatabaseMetadataGetter(cta::rdbms::Conn &conn); cta::rdbms::Login::DbType getDbType() override; - virtual ~MySQLCatalogueMetadataGetter(); + virtual ~MySQLDatabaseMetadataGetter(); }; /** - * Specific Postgres Catalogue metadata getter + * Specific Postgres database metadata getter */ -class PostgresCatalogueMetadataGetter: public CatalogueMetadataGetter{ +class PostgresDatabaseMetadataGetter: public DatabaseMetadataGetter{ public: - PostgresCatalogueMetadataGetter(cta::rdbms::Conn &conn); + PostgresDatabaseMetadataGetter(cta::rdbms::Conn &conn); cta::rdbms::Login::DbType getDbType() override; - virtual ~PostgresCatalogueMetadataGetter(); + virtual ~PostgresDatabaseMetadataGetter(); }; /** * Factory of MetadataGetter allowing to instanciate the correct metadata getter according to the database type given in parameter * @param dbType the database type in order to instanciate the correct metadata getter * @param conn the connection of the database to get the metadata from - * @return the correct CatalogueMetadataGetter instance + * @return the correct DatabaseMetadataGetter instance */ -class CatalogueMetadataGetterFactory { +class DatabaseMetadataGetterFactory { public: - static CatalogueMetadataGetter* create(const rdbms::Login::DbType dbType, cta::rdbms::Conn & conn); + static DatabaseMetadataGetter* create(const rdbms::Login::DbType dbType, cta::rdbms::Conn & conn); }; /** @@ -122,11 +122,11 @@ public: */ class SchemaMetadataGetter: public MetadataGetter{ protected: - std::unique_ptr<SQLiteCatalogueMetadataGetter> m_sqliteCatalogueMetadataGetter; + std::unique_ptr<SQLiteDatabaseMetadataGetter> m_sqliteDatabaseMetadataGetter; //The database type we would like to compare the SQLite schema against (used for filtering the results) cta::rdbms::Login::DbType m_dbType; public: - SchemaMetadataGetter(std::unique_ptr<SQLiteCatalogueMetadataGetter> sqliteCatalogueMetadataGetter, const cta::rdbms::Login::DbType dbType); + SchemaMetadataGetter(std::unique_ptr<SQLiteDatabaseMetadataGetter> sqliteCatalogueMetadataGetter, const cta::rdbms::Login::DbType dbType); virtual std::list<std::string> getIndexNames() override; virtual std::list<std::string> getTableNames() override; virtual std::map<std::string,std::string> getColumns(const std::string& tableName) override; diff --git a/catalogue/SQLiteSchemaComparer.cpp b/catalogue/SQLiteSchemaComparer.cpp index 8d446750cb..50dd9fb37e 100644 --- a/catalogue/SQLiteSchemaComparer.cpp +++ b/catalogue/SQLiteSchemaComparer.cpp @@ -25,14 +25,14 @@ namespace cta { namespace catalogue { -SQLiteSchemaComparer::SQLiteSchemaComparer(CatalogueMetadataGetter &catalogueMetadataGetter): SchemaComparer(catalogueMetadataGetter) { +SQLiteSchemaComparer::SQLiteSchemaComparer(const std::string databaseToCheckName, DatabaseMetadataGetter &catalogueMetadataGetter): SchemaComparer(databaseToCheckName,catalogueMetadataGetter) { log::DummyLogger dl("dummy","dummy"); auto login = rdbms::Login::parseString("in_memory"); //Create SQLite connection m_sqliteConnPool.reset(new rdbms::ConnPool(login,1)); m_sqliteConn = std::move(m_sqliteConnPool->getConn()); //Create the Metadata getter - std::unique_ptr<SQLiteCatalogueMetadataGetter> sqliteCatalogueMetadataGetter(new SQLiteCatalogueMetadataGetter(m_sqliteConn)); + std::unique_ptr<SQLiteDatabaseMetadataGetter> sqliteCatalogueMetadataGetter(new SQLiteDatabaseMetadataGetter(m_sqliteConn)); //Create the Schema Metadata Getter that will filter the SQLite schema metadata according to the catalogue database type we would like to compare m_schemaMetadataGetter.reset(new SchemaMetadataGetter(std::move(sqliteCatalogueMetadataGetter),catalogueMetadataGetter.getDbType())); } @@ -53,18 +53,22 @@ SchemaComparerResult SQLiteSchemaComparer::compareAll(){ SchemaComparerResult SQLiteSchemaComparer::compareTables(){ insertSchemaInSQLite(); - std::list<std::string> catalogueTables = m_catalogueMetadataGetter.getTableNames(); + std::list<std::string> catalogueTables = m_databaseMetadataGetter.getTableNames(); std::list<std::string> schemaTables = m_schemaMetadataGetter->getTableNames(); SchemaComparerResult res = compareTables(catalogueTables,schemaTables); return res; } -SchemaComparerResult SQLiteSchemaComparer::compareTablesInList(const std::list<std::string> tableNamesToCompare){ +SchemaComparerResult SQLiteSchemaComparer::compareTablesLocatedInSchema(){ insertSchemaInSQLite(); - std::list<std::string> catalogueTables = m_catalogueMetadataGetter.getTableNames(); + std::list<std::string> databaseTables = m_databaseMetadataGetter.getTableNames(); std::list<std::string> schemaTables = m_schemaMetadataGetter->getTableNames(); - SchemaComparerResult res = compareTables(catalogueTables,schemaTables); - //TODO use the list of table names to compare + databaseTables.remove_if([schemaTables](const std::string& catalogueTable){ + return std::find_if(schemaTables.begin(),schemaTables.end(),[catalogueTable](const std::string& schemaTable){ + return schemaTable == catalogueTable; + }) == schemaTables.end(); + }); + SchemaComparerResult res = compareTables(databaseTables,schemaTables); return res; } @@ -82,48 +86,48 @@ void SQLiteSchemaComparer::insertSchemaInSQLite() { SchemaComparerResult SQLiteSchemaComparer::compareIndexes(){ insertSchemaInSQLite(); - std::list<std::string> catalogueIndexes = m_catalogueMetadataGetter.getIndexNames(); + std::list<std::string> catalogueIndexes = m_databaseMetadataGetter.getIndexNames(); std::list<std::string> schemaIndexes = m_schemaMetadataGetter->getIndexNames(); return compareItems("INDEX", catalogueIndexes, schemaIndexes); } -SchemaComparerResult SQLiteSchemaComparer::compareItems(const std::string &itemType, const std::list<std::string>& itemsFromCatalogue, const std::list<std::string>& itemsFromSQLite){ +SchemaComparerResult SQLiteSchemaComparer::compareItems(const std::string &itemType, const std::list<std::string>& itemsFromDatabase, const std::list<std::string>& itemsFromSQLite){ SchemaComparerResult result; - for(auto &catalogueItem: itemsFromCatalogue){ - if(std::find(itemsFromSQLite.begin(),itemsFromSQLite.end(),catalogueItem) == itemsFromSQLite.end()){ - result.addDiff(itemType+" "+catalogueItem+" is missing in the schema but defined in the catalogue"); + for(auto &databaseItem: itemsFromDatabase){ + if(std::find(itemsFromSQLite.begin(),itemsFromSQLite.end(),databaseItem) == itemsFromSQLite.end()){ + result.addDiff(itemType+" "+databaseItem+" is missing in the schema but defined in the "+m_databaseToCheckName+" database."); } } for(auto &sqliteItem: itemsFromSQLite){ - if(std::find(itemsFromCatalogue.begin(),itemsFromCatalogue.end(),sqliteItem) == itemsFromCatalogue.end()){ - result.addDiff(itemType+" "+sqliteItem+" is missing in the catalogue but is defined in the schema"); + if(std::find(itemsFromDatabase.begin(),itemsFromDatabase.end(),sqliteItem) == itemsFromDatabase.end()){ + result.addDiff(itemType+" "+sqliteItem+" is missing in the "+m_databaseToCheckName+" database but is defined in the schema."); } } return result; } -SchemaComparerResult SQLiteSchemaComparer::compareTables(const std::list<std::string>& catalogueTables, const std::list<std::string>& schemaTables){ +SchemaComparerResult SQLiteSchemaComparer::compareTables(const std::list<std::string>& databaseTables, const std::list<std::string>& schemaTables){ SchemaComparerResult result; - std::map<std::string, std::map<std::string, std::string>> catalogueTableColumns; + std::map<std::string, std::map<std::string, std::string>> databaseTableColumns; std::map<std::string, std::map<std::string, std::string>> schemaTableColumns; - std::map<std::string,std::list<std::string>> catalogueTableConstraints; + std::map<std::string,std::list<std::string>> databaseTableConstraints; std::map<std::string, std::list<std::string>> schemaTableConstraints; - for(auto &catalogueTable: catalogueTables){ - catalogueTableColumns[catalogueTable] = m_catalogueMetadataGetter.getColumns(catalogueTable); - catalogueTableConstraints[catalogueTable] = m_catalogueMetadataGetter.getConstraintNames(catalogueTable); + for(auto &databaseTable: databaseTables){ + databaseTableColumns[databaseTable] = m_databaseMetadataGetter.getColumns(databaseTable); + databaseTableConstraints[databaseTable] = m_databaseMetadataGetter.getConstraintNames(databaseTable); } for(auto &schemaTable: schemaTables){ schemaTableColumns[schemaTable] = m_schemaMetadataGetter->getColumns(schemaTable); if(m_compareTableConstraints) { schemaTableConstraints[schemaTable] = m_schemaMetadataGetter->getConstraintNames(schemaTable); - result += compareItems("IN TABLE "+schemaTable+", CONSTRAINT",catalogueTableConstraints[schemaTable],schemaTableConstraints[schemaTable]); + result += compareItems("IN TABLE "+schemaTable+", CONSTRAINT",databaseTableConstraints[schemaTable],schemaTableConstraints[schemaTable]); } } - result += compareTableColumns(catalogueTableColumns,"catalogue",schemaTableColumns,"schema"); - result += compareTableColumns(schemaTableColumns,"schema",catalogueTableColumns,"catalogue"); + result += compareTableColumns(databaseTableColumns,m_databaseToCheckName+" database",schemaTableColumns,"schema"); + result += compareTableColumns(schemaTableColumns,"schema",databaseTableColumns,m_databaseToCheckName+" database"); return result; } diff --git a/catalogue/SQLiteSchemaComparer.hpp b/catalogue/SQLiteSchemaComparer.hpp index b6c1b48b7e..2f146938b7 100644 --- a/catalogue/SQLiteSchemaComparer.hpp +++ b/catalogue/SQLiteSchemaComparer.hpp @@ -29,10 +29,11 @@ class SQLiteSchemaComparer: public SchemaComparer { public: /** * Constructs a SQLiteSchemaComparer + * @param databaseToCheckName the database name to compare. * @param catalogueMetadataGetter the catalogue metadata getter to compare the catalogue schema content with the SQLite * database one */ - SQLiteSchemaComparer(CatalogueMetadataGetter &catalogueMetadataGetter); + SQLiteSchemaComparer(const std::string databaseToCheckName, DatabaseMetadataGetter &catalogueMetadataGetter); /** * Compare the catalogue schema against the InMemory SQLite catalogue schema * @return a SchemaComparerResult object that will contain the differences if there are some @@ -40,14 +41,14 @@ public: SchemaComparerResult compareAll() override; SchemaComparerResult compareIndexes() override; SchemaComparerResult compareTables() override; - SchemaComparerResult compareTablesInList(const std::list<std::string> tableNamesToCompare) override; + SchemaComparerResult compareTablesLocatedInSchema() override; virtual ~SQLiteSchemaComparer(); private: void insertSchemaInSQLite(); - SchemaComparerResult compareItems(const std::string &itemType, const std::list<std::string>& itemsFromCatalogue, const std::list<std::string>& itemsFromSQLite); - SchemaComparerResult compareTables(const std::list<std::string> &catalogueTables, const std::list<std::string> &schemaTables); + SchemaComparerResult compareItems(const std::string &itemType, const std::list<std::string>& itemsFromDatabase, const std::list<std::string>& itemsFromSQLite); + SchemaComparerResult compareTables(const std::list<std::string> &databaseTables, const std::list<std::string> &schemaTables); typedef std::map<std::string, std::map<std::string, std::string>> TableColumns; SchemaComparerResult compareTableColumns(const TableColumns & schema1TableColumns, const std::string &schema1Type,const TableColumns & schema2TableColumns, const std::string &schema2Type); rdbms::Conn m_sqliteConn; diff --git a/catalogue/SchemaChecker.cpp b/catalogue/SchemaChecker.cpp index 0ae985e9bd..8c95687963 100644 --- a/catalogue/SchemaChecker.cpp +++ b/catalogue/SchemaChecker.cpp @@ -22,19 +22,24 @@ namespace cta{ namespace catalogue{ -SchemaChecker::SchemaChecker(rdbms::Login::DbType dbType,cta::rdbms::Conn &conn):m_dbType(dbType),m_catalogueConn(conn) { - m_catalogueMetadataGetter.reset(CatalogueMetadataGetterFactory::create(m_dbType,m_catalogueConn)); +SchemaChecker::SchemaChecker(const std::string databaseToCheckName, rdbms::Login::DbType dbType,cta::rdbms::Conn &conn):m_databaseToCheckName(databaseToCheckName),m_dbType(dbType),m_catalogueConn(conn) { + m_databaseMetadataGetter.reset(DatabaseMetadataGetterFactory::create(m_dbType,m_catalogueConn)); } SchemaChecker::~SchemaChecker() { } -SchemaChecker::Status SchemaChecker::compareSchema(){ +void SchemaChecker::checkSchemaComparerNotNull(const std::string& methodName){ if(m_schemaComparer == nullptr){ - throw cta::exception::Exception("No schema comparer used. Please specify the schema comparer by using the methods useXXXXSchemaComparer()"); + std::string exceptionMsg = "In "+methodName+", No schema comparer used. Please specify the schema comparer by using the methods useXXXXSchemaComparer()"; + throw cta::exception::Exception(exceptionMsg); } +} + +SchemaChecker::Status SchemaChecker::compareSchema(){ + checkSchemaComparerNotNull(__PRETTY_FUNCTION__); SchemaComparerResult totalResult; - std::cout << "Schema version : " << m_catalogueMetadataGetter->getCatalogueVersion().getSchemaVersion<std::string>() << std::endl; + std::cout << "Schema version : " << m_databaseMetadataGetter->getCatalogueVersion().getSchemaVersion<std::string>() << std::endl; std::cout << "Checking indexes..." << std::endl; cta::catalogue::SchemaComparerResult resIndex = m_schemaComparer->compareIndexes(); totalResult += resIndex; @@ -57,51 +62,81 @@ SchemaChecker::Status SchemaChecker::compareSchema(){ } void SchemaChecker::checkNoParallelTables(){ - std::list<std::string> parallelTables = m_catalogueMetadataGetter->getParallelTableNames(); + std::list<std::string> parallelTables = m_databaseMetadataGetter->getParallelTableNames(); for(auto& table:parallelTables) { std::cout << "WARNING : TABLE " << table << " is set as PARALLEL" << std::endl; } } void SchemaChecker::checkSchemaNotUpgrading(){ - SchemaVersion catalogueVersion = m_catalogueMetadataGetter->getCatalogueVersion(); + SchemaVersion catalogueVersion = m_databaseMetadataGetter->getCatalogueVersion(); if(catalogueVersion.getStatus<SchemaVersion::Status>() == SchemaVersion::Status::UPGRADING){ std::cout << "WARNING : The status of the schema is " << catalogueVersion.getStatus<std::string>() << ", the future version is " << catalogueVersion.getSchemaVersionNext<std::string>() << std::endl; } } -SchemaChecker::Builder::Builder(cta::rdbms::Login::DbType dbType, cta::rdbms::Conn& conn):m_dbType(dbType),m_catalogueConn(conn){ - m_catalogueMetadataGetter.reset(CatalogueMetadataGetterFactory::create(m_dbType,m_catalogueConn)); +SchemaChecker::Status SchemaChecker::compareTablesLocatedInSchema(){ + checkSchemaComparerNotNull(__PRETTY_FUNCTION__); + SchemaComparerResult res = m_schemaComparer->compareTablesLocatedInSchema(); + if(res.getStatus() == SchemaComparerResult::Status::FAILED){ + res.printDiffs(); + return SchemaChecker::Status::FAILURE; + } + return SchemaChecker::Status::OK; +} + +SchemaChecker::Status SchemaChecker::checkTableContainsColumns(const std::string& tableName, const std::list<std::string> columnNames){ + std::map<std::string, std::string> mapColumnsTypes = m_databaseMetadataGetter->getColumns(tableName); + SchemaChecker::Status status = SchemaChecker::Status::OK; + if(mapColumnsTypes.empty()){ + std::cout << "TABLE " << tableName << " does not exist." << std::endl; + return SchemaChecker::Status::FAILURE; + } + for(auto &columnName: columnNames){ + try{ + mapColumnsTypes.at(columnName); + } catch(std::out_of_range &){ + std::cout << "TABLE " << tableName << " does not contain the column " << columnName << "."<< std::endl; + status = SchemaChecker::Status::FAILURE; + } + } + return status; +} + +///////////////////////////////////////// +// SchemaChecker::Builder +///////////////////////////////////////// +SchemaChecker::Builder::Builder(const std::string databaseToCheckName, const cta::rdbms::Login::DbType dbType, cta::rdbms::Conn & conn):m_databaseToCheckName(databaseToCheckName), m_dbType(dbType),m_catalogueConn(conn){ + m_databaseMetadataGetter.reset(DatabaseMetadataGetterFactory::create(m_dbType,m_catalogueConn)); } SchemaChecker::Builder & SchemaChecker::Builder::useSQLiteSchemaComparer(){ - m_schemaComparer.reset(new SQLiteSchemaComparer(*m_catalogueMetadataGetter)); + m_schemaComparer.reset(new SQLiteSchemaComparer(m_databaseToCheckName,*m_databaseMetadataGetter)); return *this; } SchemaChecker::Builder& SchemaChecker::Builder::useDirectorySchemaReader(const std::string& allSchemasVersionsDirectory) { - m_schemaSqlStatementsReader.reset(new DirectoryVersionsSqlStatementsReader(m_dbType,m_catalogueMetadataGetter->getCatalogueVersion().getSchemaVersion<std::string>(),allSchemasVersionsDirectory)); + m_schemaSqlStatementsReader.reset(new DirectoryVersionsSqlStatementsReader(m_dbType,m_databaseMetadataGetter->getCatalogueVersion().getSchemaVersion<std::string>(),allSchemasVersionsDirectory)); return *this; } SchemaChecker::Builder& SchemaChecker::Builder::useMapStatementsReader() { - m_schemaSqlStatementsReader.reset(new MapSqlStatementsReader(m_dbType,m_catalogueMetadataGetter->getCatalogueVersion().getSchemaVersion<std::string>())); + m_schemaSqlStatementsReader.reset(new MapSqlStatementsReader(m_dbType,m_databaseMetadataGetter->getCatalogueVersion().getSchemaVersion<std::string>())); return *this; } -SchemaChecker::Builder& SchemaChecker::Builder::useStringStatementsReader() { - m_schemaSqlStatementsReader.reset(new StringSqlStatementsReader()); +SchemaChecker::Builder& SchemaChecker::Builder::useCppSchemaStatementsReader(const cta::catalogue::CatalogueSchema schema){ + m_schemaSqlStatementsReader.reset(new CppSchemaStatementsReader(schema)); return *this; } std::unique_ptr<SchemaChecker> SchemaChecker::Builder::build() { - std::unique_ptr<SchemaChecker> schemaChecker(new SchemaChecker(m_dbType,m_catalogueConn)); + std::unique_ptr<SchemaChecker> schemaChecker(new SchemaChecker(m_databaseToCheckName,m_dbType,m_catalogueConn)); if(m_schemaComparer != nullptr){ schemaChecker->m_schemaComparer = std::move(m_schemaComparer); schemaChecker->m_schemaComparer->setSchemaSqlStatementsReader(std::move(m_schemaSqlStatementsReader)); - return std::move(schemaChecker); } - throw cta::exception::Exception("SchemaChecker::Builder::build(), a SchemaComparer should be set using the useXXXXSchemaComparer() method"); + return std::move(schemaChecker); } diff --git a/catalogue/SchemaChecker.hpp b/catalogue/SchemaChecker.hpp index 90a62ff58d..b499112f61 100644 --- a/catalogue/SchemaChecker.hpp +++ b/catalogue/SchemaChecker.hpp @@ -20,6 +20,7 @@ #include "rdbms/Login.hpp" #include "rdbms/Conn.hpp" #include "SchemaComparer.hpp" +#include "CatalogueSchema.hpp" namespace cta{ namespace catalogue{ @@ -67,19 +68,29 @@ public: */ void checkSchemaNotUpgrading(); + /** + * Compare the schema tables whose names are located in the tableList parameter + * @param tableList the table names we would like to compare + * @return a Status OK or FAILURE + */ + Status compareTablesLocatedInSchema(); + + Status checkTableContainsColumns(const std::string &tableName, const std::list<std::string> columnNames); + class Builder { public: - Builder(cta::rdbms::Login::DbType dbType, cta::rdbms::Conn &conn); + Builder(const std::string databaseToCheckName, const cta::rdbms::Login::DbType dbType, cta::rdbms::Conn &conn); Builder & useSQLiteSchemaComparer(); Builder & useDirectorySchemaReader(const std::string &allSchemasVersionsDirectory); Builder & useMapStatementsReader(); - Builder & useStringStatementsReader(); + Builder & useCppSchemaStatementsReader(const cta::catalogue::CatalogueSchema schema); std::unique_ptr<SchemaChecker> build(); private: + const std::string m_databaseToCheckName; cta::rdbms::Login::DbType m_dbType; cta::rdbms::Conn &m_catalogueConn; std::unique_ptr<SchemaComparer> m_schemaComparer; - std::unique_ptr<CatalogueMetadataGetter> m_catalogueMetadataGetter; + std::unique_ptr<DatabaseMetadataGetter> m_databaseMetadataGetter; std::unique_ptr<SchemaSqlStatementsReader> m_schemaSqlStatementsReader; }; @@ -89,8 +100,9 @@ private: * @param dbType the type of the database to check against * @param conn the connection of the database to check */ - SchemaChecker(rdbms::Login::DbType dbType,cta::rdbms::Conn &conn); + SchemaChecker(const std::string databaseToCheckName, rdbms::Login::DbType dbType,cta::rdbms::Conn &conn); + const std::string m_databaseToCheckName; /** * Catalogue-to-check database type */ @@ -108,7 +120,9 @@ private: * Catalogue metadata getter that allows to query the * metadatas of the catalogue database */ - std::unique_ptr<CatalogueMetadataGetter> m_catalogueMetadataGetter; + std::unique_ptr<DatabaseMetadataGetter> m_databaseMetadataGetter; + + void checkSchemaComparerNotNull(const std::string & methodName); }; }} diff --git a/catalogue/SchemaComparer.cpp b/catalogue/SchemaComparer.cpp index 661d924c51..a8d243159c 100644 --- a/catalogue/SchemaComparer.cpp +++ b/catalogue/SchemaComparer.cpp @@ -15,18 +15,11 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. */ -/* - * File: SchemaComparer.cpp - * Author: cedric - * - * Created on December 10, 2019, 10:58 AM - */ - #include "SchemaComparer.hpp" namespace cta { namespace catalogue { -SchemaComparer::SchemaComparer(CatalogueMetadataGetter &catalogueMetadataGetter): m_catalogueMetadataGetter(catalogueMetadataGetter),m_compareTableConstraints((m_catalogueMetadataGetter.getDbType() != cta::rdbms::Login::DBTYPE_MYSQL)){} +SchemaComparer::SchemaComparer(const std::string databaseToCheckName, DatabaseMetadataGetter &catalogueMetadataGetter): m_databaseToCheckName(databaseToCheckName),m_databaseMetadataGetter(catalogueMetadataGetter),m_compareTableConstraints((m_databaseMetadataGetter.getDbType() != cta::rdbms::Login::DBTYPE_MYSQL)){} void SchemaComparer::setSchemaSqlStatementsReader(std::unique_ptr<SchemaSqlStatementsReader> schemaSqlStatementsReader){ m_schemaSqlStatementsReader = std::move(schemaSqlStatementsReader); diff --git a/catalogue/SchemaComparer.hpp b/catalogue/SchemaComparer.hpp index c3ad92b0ce..1a281d8e3f 100644 --- a/catalogue/SchemaComparer.hpp +++ b/catalogue/SchemaComparer.hpp @@ -26,7 +26,7 @@ #include "rdbms/ConnPool.hpp" #include "SchemaComparerResult.hpp" -#include "CatalogueMetadataGetter.hpp" +#include "DatabaseMetadataGetter.hpp" #include "SchemaSqlStatementsReader.hpp" namespace cta { @@ -42,7 +42,7 @@ public: * Constructs a SchemaComparer * @param catalogueMetadataGetter the catalogue metadata getter to compare the catalogue schema content */ - SchemaComparer(CatalogueMetadataGetter &catalogueMetadataGetter); + SchemaComparer(const std::string databaseToCheckName, DatabaseMetadataGetter &databaseMetadataGetter); /** * Destructor */ @@ -64,12 +64,11 @@ public: virtual SchemaComparerResult compareIndexes() = 0; /** - * Compare only the tables in the list given in parameter - * @param tableNamesToCompare the tables to compare between the Schema and the database + * Compare only the tables that are located in the schema + * This is useful when want to compare ONLY tables that are defined in a schema * @return a SchemaComparerResult that will contain the differences if there are some */ - virtual SchemaComparerResult compareTablesInList(const std::list<std::string> tableNamesToCompare) = 0; - + virtual SchemaComparerResult compareTablesLocatedInSchema() = 0; /** * Sets the way the schema sql statements will be read to do the schemas comparison * @param schemaSqlStatementsReader the reader used to get the schema sql statements in order to do schema comparison @@ -77,7 +76,8 @@ public: void setSchemaSqlStatementsReader(std::unique_ptr<SchemaSqlStatementsReader> schemaSqlStatementsReader); protected: - cta::catalogue::CatalogueMetadataGetter & m_catalogueMetadataGetter; + const std::string m_databaseToCheckName; + cta::catalogue::DatabaseMetadataGetter & m_databaseMetadataGetter; std::unique_ptr<SchemaSqlStatementsReader> m_schemaSqlStatementsReader; bool m_compareTableConstraints; }; diff --git a/catalogue/SchemaSqlStatementsReader.cpp b/catalogue/SchemaSqlStatementsReader.cpp index ca0ee3044d..f805d107d3 100644 --- a/catalogue/SchemaSqlStatementsReader.cpp +++ b/catalogue/SchemaSqlStatementsReader.cpp @@ -18,6 +18,7 @@ #include <memory> #include <fstream> +#include <iostream> #include "SchemaSqlStatementsReader.hpp" #include "SqliteCatalogueSchema.hpp" @@ -173,13 +174,13 @@ std::list<std::string> MapSqlStatementsReader::getStatements(){ } ////////////////////////////////////////////////////////////////// -// StringSqlStatementsReader +// CppSchemaStatementsReader ////////////////////////////////////////////////////////////////// -StringSqlStatementsReader::StringSqlStatementsReader():SchemaSqlStatementsReader(){} -//TODO : Refactor -std::list<std::string> StringSqlStatementsReader::getStatements(){ - return std::list<std::string>(); +CppSchemaStatementsReader::CppSchemaStatementsReader(const cta::catalogue::CatalogueSchema schema):m_schema(schema) {} + +std::list<std::string> CppSchemaStatementsReader::getStatements(){ + return getAllStatementsFromSchema(m_schema.sql); } }} diff --git a/catalogue/SchemaSqlStatementsReader.hpp b/catalogue/SchemaSqlStatementsReader.hpp index 08b4ea54fc..4f8d7147b7 100644 --- a/catalogue/SchemaSqlStatementsReader.hpp +++ b/catalogue/SchemaSqlStatementsReader.hpp @@ -19,6 +19,7 @@ #pragma once #include "rdbms/Login.hpp" +#include "CatalogueSchema.hpp" namespace cta{ namespace catalogue { @@ -84,10 +85,12 @@ private: std::string m_catalogueVersion; }; -class StringSqlStatementsReader: public SchemaSqlStatementsReader{ +class CppSchemaStatementsReader: public SchemaSqlStatementsReader{ public: - StringSqlStatementsReader(); - virtual std::list<std::string> getStatements(); + CppSchemaStatementsReader(const cta::catalogue::CatalogueSchema schema); + std::list<std::string> getStatements(); +private: + const cta::catalogue::CatalogueSchema m_schema; }; }} \ No newline at end of file diff --git a/catalogue/VerifySchemaCmd.cpp b/catalogue/VerifySchemaCmd.cpp index a5c1419c1d..cc8f70d350 100644 --- a/catalogue/VerifySchemaCmd.cpp +++ b/catalogue/VerifySchemaCmd.cpp @@ -74,8 +74,8 @@ int VerifySchemaCmd::exceptionThrowingMain(const int argc, char *const *const ar return 1; } - SchemaChecker::Builder schemaCheckerBuilder(login.dbType,conn); - std::unique_ptr<SchemaChecker> schemaChecker(schemaCheckerBuilder + SchemaChecker::Builder schemaCheckerBuilder("catalogue",login.dbType,conn); + std::unique_ptr<SchemaChecker> schemaChecker(schemaCheckerBuilder .useMapStatementsReader() .useSQLiteSchemaComparer() .build()); diff --git a/statistics/CMakeLists.txt b/statistics/CMakeLists.txt index c12f71a493..4baa8c4d92 100644 --- a/statistics/CMakeLists.txt +++ b/statistics/CMakeLists.txt @@ -26,9 +26,10 @@ set (STATISTICS_LIB_SRC_FILES StatisticsCmd.cpp StatisticsCmdLineArgs.cpp StatisticsCmdMain.cpp + StatisticsSchema.cpp ) -add_library (ctastatistics +add_library (ctastatistics SHARED ${STATISTICS_LIB_SRC_FILES}) set_property(TARGET ctastatistics PROPERTY SOVERSION "${CTA_SOVERSION}") @@ -42,6 +43,27 @@ target_link_libraries (ctastatistics ctaschemachecker ctardbms) +add_custom_command (OUTPUT mysql_statistics_schema.sql +COMMAND cat + ${CMAKE_CURRENT_SOURCE_DIR}/common_statistics_schema.sql + | sed 's/UINT8TYPE/TINYINT UNSIGNED/g' + | sed 's/UINT16TYPE/SMALLINT UNSIGNED/g' + | sed 's/UINT32TYPE/INT UNSIGNED/g' + | sed 's/UINT64TYPE/BIGINT UNSIGNED/g' + | sed 's/CHECKSUM_BLOB_TYPE/VARBINARY\(200\)/g' + | tee mysql_statistics_schema.sql > /dev/null +DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/common_statistics_schema.sql +) + +# For Mysql +add_custom_command(OUTPUT mysql_statistics_schema.cpp + COMMAND sed 's/^/\ \ \"/' mysql_statistics_schema.sql | sed 's/$$/\"/' > mysql_statistics_schema.cpp + DEPENDS mysql_statistics_schema.sql) + +add_custom_command(OUTPUT MysqlStatisticsSchema.cpp + COMMAND sed -e '/CTA_SQL_SCHEMA/r mysql_statistics_schema.cpp' ${CMAKE_CURRENT_SOURCE_DIR}/MysqlStatisticsSchema.before_SQL.cpp > MysqlStatisticsSchema.cpp + DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/MysqlStatisticsSchema.before_SQL.cpp mysql_statistics_schema.cpp) + install (FILES ${CMAKE_SOURCE_DIR}/catalogue/cta-catalogue.conf.example DESTINATION ${CMAKE_INSTALL_SYSCONFDIR}/cta PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ) @@ -50,13 +72,15 @@ add_executable(cta-statistics StatisticsCmd.cpp StatisticsCmdLineArgs.cpp StatisticsCmdMain.cpp + MysqlStatisticsSchema.cpp ) -target_link_libraries (cta-statistics ctacatalogue ctaschemachecker) -set_property(TARGET cta-statistics APPEND PROPERTY INSTALL_RPATH ${ORACLE-INSTANTCLIENT_RPATH}) +target_link_libraries (cta-statistics ctastatistics ctacatalogue ctaschemachecker) + +set_property(TARGET cta-statistics APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH}) +set_property(TARGET cta-catalogue-schema-create APPEND PROPERTY INSTALL_RPATH ${ORACLE-INSTANTCLIENT_RPATH}) install (TARGETS cta-statistics DESTINATION /usr/bin) -install (FILES ${CMAKE_CURRENT_SOURCE_DIR}/cta-statistics.1cta DESTINATION /usr/share/man/man1) set (STATISTICS_CMD_LINE_UNIT_TESTS_LIB_SRC_FILES StatisticsCmdLineArgsTest.cpp diff --git a/statistics/MysqlStatisticsSchema.before_SQL.cpp b/statistics/MysqlStatisticsSchema.before_SQL.cpp new file mode 100644 index 0000000000..8802b75019 --- /dev/null +++ b/statistics/MysqlStatisticsSchema.before_SQL.cpp @@ -0,0 +1,33 @@ +/* + * The CERN Tape Archive (CTA) project + * Copyright (C) 2015 CERN + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include "statistics/MysqlStatisticsSchema.hpp" + +namespace cta { +namespace statistics { + +//------------------------------------------------------------------------------ +// constructor +//------------------------------------------------------------------------------ +MysqlStatisticsSchema::MysqlStatisticsSchema(): StatisticsSchema( + // CTA_SQL_SCHEMA - The contents of mysql_statistics_schema.cpp go here + ) { +} + +} // namespace statistics +} // namespace cta diff --git a/statistics/MysqlStatisticsSchema.hpp b/statistics/MysqlStatisticsSchema.hpp new file mode 100644 index 0000000000..d9f5313e12 --- /dev/null +++ b/statistics/MysqlStatisticsSchema.hpp @@ -0,0 +1,37 @@ +/* + * The CERN Tape Archive (CTA) project + * Copyright (C) 2015 CERN + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#pragma once + +#include "StatisticsSchema.hpp" + +#include <string> + +namespace cta { +namespace statistics { + + +struct MysqlStatisticsSchema: public StatisticsSchema { + /** + * Constructor. + */ + MysqlStatisticsSchema(); +}; + +} // namespace statistics +} // namespace cta diff --git a/statistics/StatisticsCmd.cpp b/statistics/StatisticsCmd.cpp index ca0e8e88d9..6935f2de85 100644 --- a/statistics/StatisticsCmd.cpp +++ b/statistics/StatisticsCmd.cpp @@ -19,7 +19,9 @@ #include "rdbms/ConnPool.hpp" #include "rdbms/AutocommitMode.hpp" #include "StatisticsCmd.hpp" +#include "StatisticsSchema.hpp" #include "catalogue/SchemaChecker.hpp" +#include "MysqlStatisticsSchema.hpp" namespace cta { @@ -42,15 +44,13 @@ StatisticsCmd::~StatisticsCmd() noexcept { // exceptionThrowingMain //------------------------------------------------------------------------------ int StatisticsCmd::exceptionThrowingMain(const int argc, char *const *const argv) { + using namespace cta::catalogue; const StatisticsCmdLineArgs cmdLineArgs(argc, argv); if(cmdLineArgs.help) { printUsage(m_out); return 0; } - - std::cout << cmdLineArgs.catalogueDbConfigPath << std::endl; - std::cout << cmdLineArgs.statisticsDbConfigPath << std::endl; const uint64_t maxNbConns = 1; @@ -58,16 +58,31 @@ int StatisticsCmd::exceptionThrowingMain(const int argc, char *const *const argv rdbms::ConnPool catalogueConnPool(loginCatalogue, maxNbConns); auto catalogueConn = catalogueConnPool.getConn(); - auto loginStatistics = rdbms::Login::parseFile(cmdLineArgs.statisticsDbConfigPath); + /*auto loginStatistics = rdbms::Login::parseFile(cmdLineArgs.statisticsDbConfigPath); rdbms::ConnPool statisticsConnPool(loginStatistics, maxNbConns); - auto statisticsConn = statisticsConnPool.getConn(); + auto statisticsConn = statisticsConnPool.getConn();*/ - /*cta::catalogue::SchemaChecker catalogueSchemaChecker(loginCatalogue.dbType,catalogueConn); - cta::catalogue::SchemaChecker statisticsSchemaChecker(loginStatistics.dbType,statisticsConn);*/ + SchemaChecker::Builder catalogueCheckerBuilder("catalogue",loginCatalogue.dbType,catalogueConn); + std::unique_ptr<cta::catalogue::SchemaChecker> catalogueChecker; + catalogueChecker = catalogueCheckerBuilder.build(); + + SchemaChecker::Status tapeTableStatus = catalogueChecker->checkTableContainsColumns("TAPE",{"VID"}); + SchemaChecker::Status tapeFileTableStatus = catalogueChecker->checkTableContainsColumns("TAPE_FILE",{"ARCHIVE_FILE_ID"}); + SchemaChecker::Status archiveFileTableStatus = catalogueChecker->checkTableContainsColumns("ARCHIVE_FILE",{"SIZE_IN_BYTES"}); -// catalogueSchemaChecker.useSQLiteSchemaComparer(); + if(tapeTableStatus == SchemaChecker::Status::FAILURE || tapeFileTableStatus == SchemaChecker::Status::FAILURE || archiveFileTableStatus == SchemaChecker::Status::FAILURE){ + return EXIT_FAILURE; + } + + /*SchemaChecker::Builder statisticsCheckerBuilder("statistics",loginStatistics.dbType,statisticsConn); + cta::statistics::MysqlStatisticsSchema mysqlSchema; + std::unique_ptr<SchemaChecker> statisticsChecker = + statisticsCheckerBuilder.useCppSchemaStatementsReader(mysqlSchema) + .useSQLiteSchemaComparer() + .build(); + statisticsChecker->compareTablesLocatedInSchema();*/ - return 0; + return EXIT_SUCCESS; } //------------------------------------------------------------------------------ // tableExists diff --git a/statistics/StatisticsSchema.cpp b/statistics/StatisticsSchema.cpp new file mode 100644 index 0000000000..4b641fd69c --- /dev/null +++ b/statistics/StatisticsSchema.cpp @@ -0,0 +1,24 @@ +/** + * The CERN Tape Archive (CTA) project + * Copyright © 2018 CERN + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ +#include "StatisticsSchema.hpp" +#include "catalogue/CatalogueSchema.hpp" + +namespace cta { namespace statistics { + + StatisticsSchema::StatisticsSchema(const std::string &sqlSchema):CatalogueSchema(sqlSchema){} +}} \ No newline at end of file diff --git a/statistics/StatisticsSchema.hpp b/statistics/StatisticsSchema.hpp new file mode 100644 index 0000000000..1c39d13065 --- /dev/null +++ b/statistics/StatisticsSchema.hpp @@ -0,0 +1,51 @@ +/* + * The CERN Tape Archive (CTA) project + * Copyright (C) 2015 CERN + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#pragma once + +#include "catalogue/CatalogueSchema.hpp" + +#include <string> + +namespace cta { +namespace statistics { + +/** + * Structure containing the SQL to create the schema of the in memory CTA + * database. + * + * The CMakeLists.txt file of this directory instructs cmake to generate + * OracleCatalogueSchema.cpp from: + * - OracleCatalogueSchema.before_SQL.cpp + * - oracle_catalogue_schema.sql + * + * The OracleSchema.before_SQL.cpp file is not compilable and is therefore + * difficult for Integrated Developent Environments (IDEs) to handle. + * + * The purpose of this class is to help IDEs by isolating the "non-compilable" + * issues into a small cpp file. + */ +struct StatisticsSchema: public cta::catalogue::CatalogueSchema { + /** + * Constructor. + */ + StatisticsSchema(const std::string &sqlSchema); +}; + +} // namespace statistics +} // namespace cta diff --git a/statistics/common_statistics_schema.sql b/statistics/common_statistics_schema.sql new file mode 100644 index 0000000000..bc7d67482c --- /dev/null +++ b/statistics/common_statistics_schema.sql @@ -0,0 +1,5 @@ +CREATE TABLE CTA_STATISTICS( + NB_MASTER_FILES UINT64TYPE NOT NULL, + MASTER_DATA_BYTES UINT64TYPE NOT NULL, + UPDATE_TIME UINT64TYPE NOT NULL +); \ No newline at end of file -- GitLab