diff --git a/catalogue/CMakeLists.txt b/catalogue/CMakeLists.txt index 9f2d978469b0f6c3ce83b3f8f550337def80f544..b0074423852bf6b7baaf045845fdd49472c5dfba 100644 --- a/catalogue/CMakeLists.txt +++ b/catalogue/CMakeLists.txt @@ -290,6 +290,7 @@ add_executable(cta-catalogue-schema-verify SQLiteSchemaComparer.cpp DbToSQLiteStatementTransformer.cpp SchemaComparerResult.cpp + SchemaChecker.cpp CatalogueMetadataGetter.cpp VerifySchemaCmd.cpp VerifySchemaCmdLineArgs.cpp diff --git a/catalogue/CatalogueMetadataGetter.cpp b/catalogue/CatalogueMetadataGetter.cpp index 61aa00de383e7019b0998ce1642a004c33191f79..e294eee562027fd2bd33a6ada32660fc2d655eea 100644 --- a/catalogue/CatalogueMetadataGetter.cpp +++ b/catalogue/CatalogueMetadataGetter.cpp @@ -17,10 +17,19 @@ */ #include "CatalogueMetadataGetter.hpp" +#include <algorithm> namespace cta { namespace catalogue { +void CatalogueMetadataGetter::removeObjectNameContaining(std::list<std::string>& objects, const std::list<std::string> &wordsToTriggerRemoval){ + objects.remove_if([&wordsToTriggerRemoval](const std::string &object){ + return std::find_if(wordsToTriggerRemoval.begin(), wordsToTriggerRemoval.end(),[&object](const std::string &wordTriggeringRemoval){ + return object.find(wordTriggeringRemoval) != std::string::npos; + }) != wordsToTriggerRemoval.end(); + }); +} + CatalogueMetadataGetter::CatalogueMetadataGetter(cta::rdbms::Conn& conn):m_conn(conn){} std::string CatalogueMetadataGetter::getCatalogueVersion(){ @@ -52,32 +61,38 @@ SQLiteCatalogueMetadataGetter::~SQLiteCatalogueMetadataGetter(){} std::list<std::string> SQLiteCatalogueMetadataGetter::getIndexNames() { std::list<std::string> indexNames = m_conn.getIndexNames(); - indexNames.remove_if([](std::string& indexName){ - return ((indexName.find("sqlite_autoindex") != std::string::npos)); - }); + removeObjectNameContaining(indexNames,{"sqlite_autoindex"}); return indexNames; } std::list<std::string> SQLiteCatalogueMetadataGetter::getTableNames(){ - return std::list<std::string>(); + std::list<std::string> tableNames = m_conn.getTableNames(); + removeObjectNameContaining(tableNames,{"sqlite_sequence"}); + return tableNames; +} + +std::map<std::string, std::string> SQLiteCatalogueMetadataGetter::getColumns(const std::string& tableName){ + return m_conn.getColumns(tableName); } + OracleCatalogueMetadataGetter::OracleCatalogueMetadataGetter(cta::rdbms::Conn & conn):CatalogueMetadataGetter(conn){} OracleCatalogueMetadataGetter::~OracleCatalogueMetadataGetter(){} std::list<std::string> OracleCatalogueMetadataGetter::getIndexNames() { std::list<std::string> indexNames = m_conn.getIndexNames(); - indexNames.remove_if([](std::string& indexName){ - return ((indexName.find("_UN") != std::string::npos) || (indexName.find("PK") != std::string::npos) || (indexName.find("_LLN") != std::string::npos)); - }); + removeObjectNameContaining(indexNames,{"_UN","PK","_LLN"}); return indexNames; } std::list<std::string> OracleCatalogueMetadataGetter::getTableNames() { - return std::list<std::string>(); + std::list<std::string> tableNames = m_conn.getTableNames(); + return tableNames; } - +std::map<std::string, std::string> OracleCatalogueMetadataGetter::getColumns(const std::string& tableName){ + return m_conn.getColumns(tableName); +} CatalogueMetadataGetter * CatalogueMetadataGetterFactory::create(const rdbms::Login::DbType dbType, cta::rdbms::Conn & conn) { typedef rdbms::Login::DbType DbType; diff --git a/catalogue/CatalogueMetadataGetter.hpp b/catalogue/CatalogueMetadataGetter.hpp index c84738021dde82e600f4b89ff0045d7479517041..29e109536a430c7f7fdd33e52484633de82e9f13 100644 --- a/catalogue/CatalogueMetadataGetter.hpp +++ b/catalogue/CatalogueMetadataGetter.hpp @@ -27,15 +27,16 @@ namespace cta { namespace catalogue { class CatalogueMetadataGetter { -public: protected: rdbms::Conn& m_conn; + void removeObjectNameContaining(std::list<std::string>& objects, const std::list<std::string> &wordsToTriggerRemoval); public: CatalogueMetadataGetter(cta::rdbms::Conn & conn); virtual ~CatalogueMetadataGetter(); std::string getCatalogueVersion(); virtual std::list<std::string> getIndexNames() = 0; virtual std::list<std::string> getTableNames() = 0; + virtual std::map<std::string,std::string> getColumns(const std::string& tableName) = 0; }; class SQLiteCatalogueMetadataGetter: public CatalogueMetadataGetter{ @@ -44,6 +45,7 @@ public: virtual ~SQLiteCatalogueMetadataGetter(); std::list<std::string> getIndexNames() override; std::list<std::string> getTableNames() override; + std::map<std::string,std::string> getColumns(const std::string& tableName) override; }; class OracleCatalogueMetadataGetter: public CatalogueMetadataGetter{ @@ -52,6 +54,7 @@ class OracleCatalogueMetadataGetter: public CatalogueMetadataGetter{ virtual ~OracleCatalogueMetadataGetter(); std::list<std::string> getIndexNames() override; std::list<std::string> getTableNames() override; + std::map<std::string,std::string> getColumns(const std::string& tableName) override; }; class CatalogueMetadataGetterFactory { diff --git a/catalogue/SQLiteSchemaComparer.cpp b/catalogue/SQLiteSchemaComparer.cpp index 7629169db44620df8b966fa2931f26a6edc80284..52678ecd7c3f5bc87e3c7d7d1033ad9ff04859f9 100644 --- a/catalogue/SQLiteSchemaComparer.cpp +++ b/catalogue/SQLiteSchemaComparer.cpp @@ -35,6 +35,7 @@ SQLiteSchemaComparer::SQLiteSchemaComparer(const cta::rdbms::Login::DbType &cata } SQLiteSchemaComparer::~SQLiteSchemaComparer() { + m_sqliteSchemaMetadataGetter.release(); } SchemaComparerResult SQLiteSchemaComparer::compare(){ @@ -48,15 +49,8 @@ SchemaComparerResult SQLiteSchemaComparer::compare(){ SchemaComparerResult SQLiteSchemaComparer::compareTables(){ std::list<std::string> catalogueTables = m_catalogueMetadataGetter->getTableNames(); std::list<std::string> schemaTables = m_sqliteSchemaMetadataGetter->getTableNames(); - std::cout<<"catalogue tables = "<<std::endl; - for(auto& catalogueTable: catalogueTables){ - std::cout<<catalogueTable<<std::endl; - } - std::cout<<"schema tables = "<<std::endl; - for(auto& schemaTable: schemaTables){ - std::cout<<schemaTable<<std::endl; - } - return SchemaComparerResult(); + SchemaComparerResult res = compareTables(catalogueTables,schemaTables); + return res; } void SQLiteSchemaComparer::insertSchemaInSQLite() { @@ -65,17 +59,69 @@ void SQLiteSchemaComparer::insertSchemaInSQLite() { } SchemaComparerResult SQLiteSchemaComparer::compareIndexes(){ - SchemaComparerResult result; std::list<std::string> catalogueIndexes = m_catalogueMetadataGetter->getIndexNames(); std::list<std::string> schemaIndexes = m_sqliteSchemaMetadataGetter->getIndexNames(); - for(auto &ci: catalogueIndexes){ - if(std::find(schemaIndexes.begin(),schemaIndexes.end(),ci) == schemaIndexes.end()){ - result.addDiff("INDEX "+ci+" is missing in the schema but defined in the catalogue"); + return compareItems("INDEX", catalogueIndexes, schemaIndexes); +} + +SchemaComparerResult SQLiteSchemaComparer::compareItems(const std::string &itemType, const std::list<std::string>& itemsFromCatalogue, const std::list<std::string>& itemsFromSQLite){ + SchemaComparerResult result; + for(auto &catalogueItem: itemsFromCatalogue){ + if(std::find(itemsFromSQLite.begin(),itemsFromSQLite.end(),catalogueItem) == itemsFromSQLite.end()){ + result.addDiff(itemType+" "+catalogueItem+" is missing in the schema but defined in the catalogue"); + } + } + for(auto &sqliteItem: itemsFromSQLite){ + if(std::find(itemsFromCatalogue.begin(),itemsFromCatalogue.end(),sqliteItem) == itemsFromCatalogue.end()){ + result.addDiff(itemType+" "+sqliteItem+" is missing in the catalogue but is defined in the schema"); } } - for(auto &si: schemaIndexes){ - if(std::find(catalogueIndexes.begin(),catalogueIndexes.end(),si) == catalogueIndexes.end()){ - result.addDiff("INDEX "+si+" is missing in the catalogue but is defined in the schema"); + return result; +} + +SchemaComparerResult SQLiteSchemaComparer::compareTables(const std::list<std::string>& catalogueTables, const std::list<std::string>& schemaTables){ + SchemaComparerResult result; + std::map<std::string, std::map<std::string, std::string>> catalogueTableColumns; + std::map<std::string, std::map<std::string, std::string>> schemaTableColumns; + for(auto &catalogueTable: catalogueTables){ + catalogueTableColumns[catalogueTable] = m_catalogueMetadataGetter->getColumns(catalogueTable); + } + + for(auto &schemaTable: schemaTables){ + schemaTableColumns[schemaTable] = m_sqliteSchemaMetadataGetter->getColumns(schemaTable); + } + + result += compareTableColumns(catalogueTableColumns,"catalogue",schemaTableColumns,"schema"); + result += compareTableColumns(schemaTableColumns,"schema",catalogueTableColumns,"catalogue"); + + return result; +} + +SchemaComparerResult SQLiteSchemaComparer::compareTableColumns(const TableColumns & firstSchemaTableColumns, const std::string &schema1Type,const TableColumns & secondSchemaTableColumns, const std::string &schema2Type){ + SchemaComparerResult result; + for(auto &kvFirstSchemaTableColumns: firstSchemaTableColumns){ + //For each firstSchema table, get the corresponding secondSchema table + //If it does not exist, add a difference and go ahead + //otherwise, get the columns/types of the table from the firstSchema and do the comparison + //against the columns/types of the same table from the secondSchema + std::string firstSchemaTableName = kvFirstSchemaTableColumns.first; + try { + std::map<std::string, std::string> mapSecondSchemaColumnType = secondSchemaTableColumns.at(firstSchemaTableName); + std::map<std::string, std::string> mapFirstSchemaColumnType = kvFirstSchemaTableColumns.second; + for(auto &kvFirstSchemaColumn: mapFirstSchemaColumnType){ + std::string firstSchemaColumnName = kvFirstSchemaColumn.first; + std::string firstSchemaColumnType = kvFirstSchemaColumn.second; + try { + std::string schemaColumnType = mapSecondSchemaColumnType.at(firstSchemaColumnName); + if( firstSchemaColumnType != schemaColumnType){ + result.addDiff("TABLE "+firstSchemaTableName+" from "+schema1Type+" has a column named "+firstSchemaColumnName+" that has a type "+firstSchemaColumnType+" that does not match the column type from the "+schema2Type+" ("+schemaColumnType+")"); + } + } catch (const std::out_of_range &) { + result.addDiff("TABLE "+firstSchemaTableName+" from "+schema1Type+" has a column named " + firstSchemaColumnName + " that is missing in the "+schema2Type+"."); + } + } + } catch (const std::out_of_range &) { + result.addDiff("TABLE "+firstSchemaTableName+" is missing in the "+schema2Type+" but is defined in the "+schema1Type+"."); } } return result; diff --git a/catalogue/SQLiteSchemaComparer.hpp b/catalogue/SQLiteSchemaComparer.hpp index 85b20fc16cefa320ffc99f4d597fcbed8dce060e..62fa6c0c5e95312fc87edb14db514bbe6eb1f4b2 100644 --- a/catalogue/SQLiteSchemaComparer.hpp +++ b/catalogue/SQLiteSchemaComparer.hpp @@ -24,7 +24,7 @@ namespace catalogue { class SQLiteSchemaComparer: public SchemaComparer { public: - SQLiteSchemaComparer(const cta::rdbms::Login::DbType &catalogueDbType, rdbms::Conn &catalogueConnPool); + SQLiteSchemaComparer(const cta::rdbms::Login::DbType &catalogueDbType, rdbms::Conn &catalogueConn); SchemaComparerResult compare() override; virtual ~SQLiteSchemaComparer(); @@ -32,6 +32,10 @@ private: void insertSchemaInSQLite(); SchemaComparerResult compareIndexes(); SchemaComparerResult compareTables() override; + SchemaComparerResult compareItems(const std::string &itemType, const std::list<std::string>& itemsFromCatalogue, const std::list<std::string>& itemsFromSQLite); + SchemaComparerResult compareTables(const std::list<std::string> &catalogueTables, const std::list<std::string> &schemaTables); + typedef std::map<std::string, std::map<std::string, std::string>> TableColumns; + SchemaComparerResult compareTableColumns(const TableColumns & tableColumns1, const std::string &originTableColumns1,const TableColumns & tableColumns2, const std::string &originTableColumns2); rdbms::Conn m_sqliteConn; std::unique_ptr<SQLiteCatalogueMetadataGetter> m_sqliteSchemaMetadataGetter; }; diff --git a/catalogue/SchemaChecker.cpp b/catalogue/SchemaChecker.cpp new file mode 100644 index 0000000000000000000000000000000000000000..17f6c819b00b430808c27a58c9981647d0264672 --- /dev/null +++ b/catalogue/SchemaChecker.cpp @@ -0,0 +1,50 @@ +/** + * The CERN Tape Archive (CTA) project + * Copyright © 2018 CERN + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include "SchemaChecker.hpp" +#include "SQLiteSchemaComparer.hpp" + +namespace cta{ +namespace catalogue{ + +SchemaChecker::SchemaChecker(const cta::rdbms::Login::DbType &catalogueDbType,cta::rdbms::Conn &conn):m_dbType(catalogueDbType),m_conn(conn) { + m_schemaComparer.reset(new SQLiteSchemaComparer(m_dbType,m_conn)); +} + +SchemaChecker::~SchemaChecker() { +} + +void SchemaChecker::setSchemaComparer(std::unique_ptr<SchemaComparer> schemaComparer){ + if(m_schemaComparer != nullptr){ + m_schemaComparer.release(); + } + m_schemaComparer = std::move(schemaComparer); +} + +void SchemaChecker::compareSchema(){ + cta::catalogue::SchemaComparerResult res = m_schemaComparer->compare(); + std::cout << "Schema version : " << m_schemaComparer->getCatalogueVersion() << std::endl; + std::cout << "Status of the checking : " << cta::catalogue::SchemaComparerResult::StatusToString(res.getStatus()) << std::endl; + res.printDiffs(); +} + +void SchemaChecker::checkNoParallelTables(){ + +} + +}} \ No newline at end of file diff --git a/catalogue/SchemaChecker.hpp b/catalogue/SchemaChecker.hpp new file mode 100644 index 0000000000000000000000000000000000000000..13d44cc997de6729850b41cdcb31805d3f17d894 --- /dev/null +++ b/catalogue/SchemaChecker.hpp @@ -0,0 +1,42 @@ +/** + * The CERN Tape Archive (CTA) project + * Copyright © 2018 CERN + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#pragma once +#include "rdbms/Login.hpp" +#include "rdbms/Conn.hpp" +#include "SchemaComparer.hpp" + +namespace cta{ +namespace catalogue{ + +class SchemaChecker { +public: + SchemaChecker(const cta::rdbms::Login::DbType &catalogueDbType,cta::rdbms::Conn &conn); + virtual ~SchemaChecker(); + void setSchemaComparer(std::unique_ptr<SchemaComparer> schemaComparer); + void compareSchema(); + void checkNoParallelTables(); +private: + const cta::rdbms::Login::DbType &m_dbType; + cta::rdbms::Conn &m_conn; + std::unique_ptr<SchemaComparer> m_schemaComparer; +}; + +}} + + diff --git a/catalogue/SchemaComparer.hpp b/catalogue/SchemaComparer.hpp index 805efe783a8bb34bad49fd03c14684c32fb5b50c..81c9e1f52936b17515b4490fb07a4af46b5baed2 100644 --- a/catalogue/SchemaComparer.hpp +++ b/catalogue/SchemaComparer.hpp @@ -35,6 +35,10 @@ class SchemaComparer { public: SchemaComparer(const cta::rdbms::Login::DbType &catalogueDbType,cta::rdbms::Conn &conn); virtual ~SchemaComparer(); + /** + * Compare the schema + * @return + */ virtual SchemaComparerResult compare() = 0; std::string getCatalogueVersion(); protected: diff --git a/catalogue/VerifySchemaCmd.cpp b/catalogue/VerifySchemaCmd.cpp index 8cc02e7f865a091ff942700574d44d56a16f6901..ddd678f596005fd50fd7ea928517fa4bce527e41 100644 --- a/catalogue/VerifySchemaCmd.cpp +++ b/catalogue/VerifySchemaCmd.cpp @@ -29,8 +29,7 @@ #include "rdbms/AutocommitMode.hpp" #include "common/log/DummyLogger.hpp" #include "Catalogue.hpp" -#include "SchemaComparer.hpp" -#include "SQLiteSchemaComparer.hpp" +#include "SchemaChecker.hpp" #include <algorithm> #include <map> @@ -74,13 +73,10 @@ int VerifySchemaCmd::exceptionThrowingMain(const int argc, char *const *const ar return 1; } - std::unique_ptr<cta::catalogue::SchemaComparer> schemaComparer; - schemaComparer.reset(new cta::catalogue::SQLiteSchemaComparer(login.dbType,conn)); - cta::catalogue::SchemaComparerResult res = schemaComparer->compare(); - std::cout << "Schema version : " << schemaComparer->getCatalogueVersion() << std::endl; - std::cout << "Status of the checking : " << cta::catalogue::SchemaComparerResult::StatusToString(res.getStatus()) << std::endl; - res.printDiffs(); - schemaComparer.release(); + cta::catalogue::SchemaChecker schemaChecker(login.dbType,conn); + schemaChecker.compareSchema(); + schemaChecker.checkNoParallelTables(); + /*std::unique_ptr<CatalogueSchema> schema; switch(login.dbType) { case rdbms::Login::DBTYPE_IN_MEMORY: diff --git a/rdbms/wrapper/SqliteConn.cpp b/rdbms/wrapper/SqliteConn.cpp index 0b230de140033fcf964bb14412a3ba3db45f2149..cd1be217d786f44f12e892e7034c285b77e7be36 100644 --- a/rdbms/wrapper/SqliteConn.cpp +++ b/rdbms/wrapper/SqliteConn.cpp @@ -249,7 +249,8 @@ std::map<std::string, std::string> SqliteConn::getColumns(const std::string &tab "CHAR|" "VARCHAR|" "VARCHAR2|" - "BLOB"; + "BLOB|" + "RAW"; auto stmt = createStmt(sql); stmt->bindString(":TABLE_NAME", tableName);