Commit 68d58713 authored by Cedric CAFFY's avatar Cedric CAFFY
Browse files

cta-catalogue-schema-verify indexes comparison done

parent 15aafada
......@@ -289,6 +289,8 @@ add_executable(cta-catalogue-schema-verify
SchemaComparer.cpp
SQLiteSchemaComparer.cpp
DbToSQLiteStatementTransformer.cpp
SchemaComparerResult.cpp
CatalogueMetadataGetter.cpp
VerifySchemaCmd.cpp
VerifySchemaCmdLineArgs.cpp
VerifySchemaCmdMain.cpp
......
/**
* The CERN Tape Archive (CTA) project
* Copyright © 2018 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "CatalogueMetadataGetter.hpp"
namespace cta {
namespace catalogue {
CatalogueMetadataGetter::CatalogueMetadataGetter(cta::rdbms::ConnPool & connPool):m_connPool(connPool){}
CatalogueMetadataGetter::~CatalogueMetadataGetter() {}
SQLiteCatalogueMetadataGetter::SQLiteCatalogueMetadataGetter(cta::rdbms::ConnPool & connPool):CatalogueMetadataGetter(connPool){}
SQLiteCatalogueMetadataGetter::~SQLiteCatalogueMetadataGetter(){}
std::list<std::string> SQLiteCatalogueMetadataGetter::getIndexNames() {
rdbms::Conn connection = m_connPool.getConn();
std::list<std::string> indexNames = connection.getIndexNames();
connection.closeUnderlyingStmtsAndConn();
indexNames.remove_if([](std::string& indexName){
return ((indexName.find("sqlite_autoindex") != std::string::npos));
});
return indexNames;
}
OracleCatalogueMetadataGetter::OracleCatalogueMetadataGetter(cta::rdbms::ConnPool & connPool):CatalogueMetadataGetter(connPool){}
OracleCatalogueMetadataGetter::~OracleCatalogueMetadataGetter(){}
std::list<std::string> OracleCatalogueMetadataGetter::getIndexNames() {
rdbms::Conn connection = m_connPool.getConn();
std::list<std::string> indexNames = connection.getIndexNames();
connection.closeUnderlyingStmtsAndConn();
indexNames.remove_if([](std::string& indexName){
return ((indexName.find("_UN") != std::string::npos) || (indexName.find("PK") != std::string::npos) || (indexName.find("_LLN") != std::string::npos));
});
return indexNames;
}
CatalogueMetadataGetter * CatalogueMetadataGetterFactory::create(const rdbms::Login::DbType dbType, cta::rdbms::ConnPool & connPool) {
typedef rdbms::Login::DbType DbType;
switch(dbType){
case DbType::DBTYPE_IN_MEMORY:
case DbType::DBTYPE_SQLITE:
return new SQLiteCatalogueMetadataGetter(connPool);
case DbType::DBTYPE_ORACLE:
return new OracleCatalogueMetadataGetter(connPool);
default:
return nullptr;
}
}
}}
\ No newline at end of file
/**
* The CERN Tape Archive (CTA) project
* Copyright © 2018 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "rdbms/Conn.hpp"
#include "rdbms/Login.hpp"
#include "SchemaCreatingSqliteCatalogue.hpp"
#include <list>
namespace cta {
namespace catalogue {
class CatalogueMetadataGetter {
public:
protected:
rdbms::ConnPool& m_connPool;
public:
CatalogueMetadataGetter(cta::rdbms::ConnPool & connPool);
virtual ~CatalogueMetadataGetter();
virtual std::list<std::string> getIndexNames() = 0;
};
class SQLiteCatalogueMetadataGetter: public CatalogueMetadataGetter{
public:
SQLiteCatalogueMetadataGetter(cta::rdbms::ConnPool & connPool);
virtual ~SQLiteCatalogueMetadataGetter();
std::list<std::string> getIndexNames() override;
};
class OracleCatalogueMetadataGetter: public CatalogueMetadataGetter{
public:
OracleCatalogueMetadataGetter(cta::rdbms::ConnPool & connPool);
virtual ~OracleCatalogueMetadataGetter();
std::list<std::string> getIndexNames() override;
};
class CatalogueMetadataGetterFactory {
public:
static CatalogueMetadataGetter* create(const rdbms::Login::DbType dbType, cta::rdbms::ConnPool & connPool);
};
}}
\ No newline at end of file
......@@ -25,6 +25,7 @@
namespace cta {
namespace catalogue {
/**********************************/
/* DbToSQLiteStatementTransformer */
/**********************************/
......@@ -38,14 +39,29 @@ std::string DbToSQLiteStatementTransformer::transform(){
return m_statement;
}
/**********************************/
/* DbToSQLiteStatementTransformer */
/**********************************/
/*****************************************************/
/* CreateGlobalTempTableToSQLiteStatementTransformer */
/*****************************************************/
CreateGlobalTempTableToSQLiteStatementTransformer::CreateGlobalTempTableToSQLiteStatementTransformer(const std::string &statement):DbToSQLiteStatementTransformer(statement){}
std::string CreateGlobalTempTableToSQLiteStatementTransformer::transform(){
utils::searchAndReplace(m_statement,"GLOBAL TEMPORARY ","");
utils::searchAndReplace(m_statement,"ON COMMIT DELETE ROWS;",";");
utils::searchAndReplace(m_statement,"\nON COMMIT DELETE ROWS;",";");
return m_statement;
}
/*****************************************************/
/* CreateGlobalTempTableToSQLiteStatementTransformer */
/*****************************************************/
/**********************************/
/* DeleteStatementTransformer */
/**********************************/
DeleteStatementTransformer::DeleteStatementTransformer(const std::string& statement):DbToSQLiteStatementTransformer(statement){}
......@@ -53,6 +69,13 @@ std::string DeleteStatementTransformer::transform(){
return "";
}
/**********************************/
/* DeleteStatementTransformer */
/**********************************/
/*****************************************************/
/* DbToSQLiteStatementTransformerFactory */
/*****************************************************/
std::unique_ptr<DbToSQLiteStatementTransformer> DbToSQLiteStatementTransformerFactory::create(const std::string &statement){
typedef DbToSQLiteStatementTransformerFactory::StatementType StatementType;
StatementType stmtType = statementToStatementType(statement);
......@@ -102,4 +125,8 @@ std::map<std::string,DbToSQLiteStatementTransformerFactory::StatementType> DbToS
return ret;
}
/*****************************************************/
/* DbToSQLiteStatementTransformerFactory */
/*****************************************************/
}}
\ No newline at end of file
......@@ -15,12 +15,12 @@
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <string>
#include <map>
#include <memory>
#pragma once
namespace cta {
namespace catalogue {
......
......@@ -15,37 +15,59 @@
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
/*
* File: SQLiteSchemaComparer.cpp
* Author: cedric
*
* Created on December 10, 2019, 11:34 AM
*/
#include <algorithm>
#include "SQLiteSchemaComparer.hpp"
#include "SQLiteSchemaInserter.hpp"
#include "common/log/DummyLogger.hpp"
namespace cta {
namespace catalogue {
SQLiteSchemaComparer::SQLiteSchemaComparer(cta::rdbms::Conn &connection, cta::rdbms::Login &login, cta::catalogue::Catalogue& catalogue): SchemaComparer(connection,login,catalogue) {
SQLiteSchemaComparer::SQLiteSchemaComparer(const cta::rdbms::Login::DbType &catalogueDbType, const std::string &schemaVersion, rdbms::ConnPool &catalogueConnPool): SchemaComparer(catalogueDbType,schemaVersion,catalogueConnPool) {
log::DummyLogger dl("dummy","dummy");
auto login = rdbms::Login::parseString("in_memory");
m_sqliteConnPool.reset(new rdbms::ConnPool(login,1));
m_catalogueMetadataGetter.reset(CatalogueMetadataGetterFactory::create(catalogueDbType,catalogueConnPool));
m_sqliteSchemaMetadataGetter.reset(new SQLiteCatalogueMetadataGetter(*m_sqliteConnPool));
}
SQLiteSchemaComparer::~SQLiteSchemaComparer() {
}
void SQLiteSchemaComparer::compare(){
SchemaComparerResult SQLiteSchemaComparer::compare(){
SchemaComparerResult res;
insertSchemaInSQLite();
//compare
//return the results
res += compareIndexes();
return res;
}
SchemaComparerResult SQLiteSchemaComparer::compareTables(){
return SchemaComparerResult();
}
void SQLiteSchemaComparer::insertSchemaInSQLite() {
std::map<std::string, uint64_t> schemaVersion = m_catalogue.getSchemaVersion();
uint64_t schemaVersionMajor = schemaVersion.at("SCHEMA_VERSION_MAJOR");
uint64_t schemaVersionMinor = schemaVersion.at("SCHEMA_VERSION_MINOR");
std::string schemaVersionStr = std::to_string(schemaVersionMajor)+"."+std::to_string(schemaVersionMinor);
cta::catalogue::SQLiteSchemaInserter schemaInserter(schemaVersionStr,m_login.dbType,"/home/cedric/CTA/catalogue/","/home/cedric/CTA-execute/sqliteDbFile");
cta::catalogue::SQLiteSchemaInserter schemaInserter(m_schemaVersion,m_dbType,"/home/cedric/CTA/catalogue/",*m_sqliteConnPool);
schemaInserter.insert();
}
SchemaComparerResult SQLiteSchemaComparer::compareIndexes(){
SchemaComparerResult result;
std::list<std::string> catalogueIndexes = m_catalogueMetadataGetter->getIndexNames();
std::list<std::string> schemaIndexes = m_sqliteSchemaMetadataGetter->getIndexNames();
for(auto &ci: catalogueIndexes){
if(std::find(schemaIndexes.begin(),schemaIndexes.end(),ci) == schemaIndexes.end()){
result.addDiff("INDEX "+ci+" is missing in the schema but defined in the catalogue");
}
}
for(auto &si: schemaIndexes){
if(std::find(catalogueIndexes.begin(),catalogueIndexes.end(),si) == catalogueIndexes.end()){
result.addDiff("INDEX "+si+" is missing in the catalogue but is defined in the schema");
}
}
return result;
}
}}
\ No newline at end of file
......@@ -15,20 +15,27 @@
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "SchemaComparer.hpp"
#pragma once
#include "SchemaComparer.hpp"
#include "SchemaCreatingSqliteCatalogue.hpp"
#include "InMemoryCatalogue.hpp"
namespace cta {
namespace catalogue {
class SQLiteSchemaComparer: public SchemaComparer {
public:
SQLiteSchemaComparer(cta::rdbms::Conn &connection, cta::rdbms::Login &login, cta::catalogue::Catalogue& catalogue);
void compare() override;
SQLiteSchemaComparer(const cta::rdbms::Login::DbType &catalogueDbType, const std::string &schemaVersion,rdbms::ConnPool &catalogueConnPool);
SchemaComparerResult compare() override;
virtual ~SQLiteSchemaComparer();
private:
void insertSchemaInSQLite();
SchemaComparerResult compareIndexes();
SchemaComparerResult compareTables() override;
std::unique_ptr<rdbms::ConnPool> m_sqliteConnPool;
std::unique_ptr<SQLiteCatalogueMetadataGetter> m_sqliteSchemaMetadataGetter;
};
}}
\ No newline at end of file
......@@ -27,31 +27,27 @@
namespace cta {
namespace catalogue {
SQLiteSchemaInserter::SQLiteSchemaInserter(const std::string & schemaVersion, const cta::rdbms::Login::DbType &dbType, const std::string &allVersionsSchemaDirectory,const std::string &sqliteFileName):m_schemaVersion(schemaVersion),m_dbType(dbType),m_allVersionSchemaDirectory(allVersionsSchemaDirectory),m_sqliteFileName(sqliteFileName) {
log::DummyLogger dl("dummy","dummy");
m_sqliteCatalogue = new SchemaCreatingSqliteCatalogue(dl,m_sqliteFileName,1,1,false);
m_conn = m_sqliteCatalogue->m_connPool.getConn();
}
SQLiteSchemaInserter::~SQLiteSchemaInserter() {
if(m_sqliteCatalogue != nullptr) {
delete m_sqliteCatalogue;
}
::remove(m_sqliteFileName.c_str());
}
SQLiteSchemaInserter::SQLiteSchemaInserter(const std::string & schemaVersion, const cta::rdbms::Login::DbType &catalogueDbType, const std::string &allVersionsSchemaDirectory,rdbms::ConnPool &sqliteConnPool):m_schemaVersion(schemaVersion),m_dbType(catalogueDbType),m_allVersionSchemaDirectory(allVersionsSchemaDirectory), m_sqliteCatalogueConnPool(sqliteConnPool){}
SQLiteSchemaInserter::~SQLiteSchemaInserter() {}
void SQLiteSchemaInserter::insert() {
std::list<std::string> statements = getAllStatementsFromSchema(readSchemaFromFile());
std::string transformedSchema;
std::list<std::string> sqliteStatements;
for(auto& stmt: statements){
transformedSchema.append(DbToSQLiteStatementTransformerFactory::create(stmt)->transform());
std::string sqliteStatement = DbToSQLiteStatementTransformerFactory::create(stmt)->transform();
if(!sqliteStatement.empty())
sqliteStatements.emplace_back(sqliteStatement);
}
m_sqliteCatalogue->executeNonQueries(m_conn,transformedSchema);
executeStatements(sqliteStatements);
}
void SQLiteSchemaInserter::executeStatements(const std::string &sqliteStatements){
void SQLiteSchemaInserter::executeStatements(const std::list<std::string> & stmts){
rdbms::Conn conn = m_sqliteCatalogueConnPool.getConn();
for(auto& sqliteStatement: stmts){
auto stmt = conn.createStmt(sqliteStatement);
stmt.executeNonQuery();
}
}
std::list<std::string> SQLiteSchemaInserter::getAllStatementsFromSchema(const std::string &schema){
......
......@@ -18,30 +18,28 @@
#pragma once
#include "rdbms/Login.hpp"
#include "SchemaCreatingSqliteCatalogue.hpp"
#include "rdbms/ConnPool.hpp"
namespace cta {
namespace catalogue {
class SQLiteSchemaInserter {
public:
SQLiteSchemaInserter(const std::string & schemaVersion, const cta::rdbms::Login::DbType &dbType, const std::string &allVersionsSchemaDirectory,const std::string &sqliteFileName);
SQLiteSchemaInserter(const std::string & schemaVersion, const cta::rdbms::Login::DbType &catalogueDbType, const std::string &allVersionsSchemaDirectory,rdbms::ConnPool &sqliteConnPool);
void insert();
virtual ~SQLiteSchemaInserter();
private:
const std::string c_catalogueFileNameTrailer = "_catalogue_schema.sql";
std::string m_schemaVersion;
cta::rdbms::Login::DbType m_dbType;
std::string m_allVersionSchemaDirectory;
const std::string c_catalogueFileNameTrailer = "_catalogue_schema.sql";
std::string m_sqliteFileName;
cta::catalogue::SchemaCreatingSqliteCatalogue * m_sqliteCatalogue;
cta::rdbms::Conn m_conn;
cta::rdbms::ConnPool & m_sqliteCatalogueConnPool;
std::string readSchemaFromFile();
std::list<std::string> getAllStatementsFromSchema(const std::string &schema);
std::string getDatabaseType();
std::string getSchemaFilePath();
void executeStatements(const std::string &statements);
void executeStatements(const std::list<std::string> &statements);
};
}}
......
......@@ -26,9 +26,15 @@
namespace cta {
namespace catalogue {
SchemaComparer::SchemaComparer(cta::rdbms::Conn &connection, cta::rdbms::Login &login, cta::catalogue::Catalogue& catalogue):m_conn(connection), m_login(login), m_catalogue(catalogue){
SchemaComparer::SchemaComparer(const cta::rdbms::Login::DbType &dbType, const std::string &schemaVersion,rdbms::ConnPool &connPool): m_dbType(dbType),m_schemaVersion(schemaVersion),m_catalogueConnPool(connPool){
m_catalogueMetadataGetter.reset(CatalogueMetadataGetterFactory::create(dbType,m_catalogueConnPool));
}
std::string SchemaComparer::getCatalogueVersion(){
return m_schemaVersion;
}
SchemaComparer::~SchemaComparer() {
}
}}
......@@ -24,22 +24,27 @@
#pragma once
#include "rdbms/Conn.hpp"
#include "rdbms/Login.hpp"
#include "catalogue/Catalogue.hpp"
#include "rdbms/ConnPool.hpp"
#include "SchemaComparerResult.hpp"
#include "CatalogueMetadataGetter.hpp"
namespace cta {
namespace catalogue {
class SchemaComparer {
public:
SchemaComparer(cta::rdbms::Conn &connection, cta::rdbms::Login &login, cta::catalogue::Catalogue& catalogue);
SchemaComparer(const cta::rdbms::Login::DbType &catalogueDbType,const std::string &schemaVersion,cta::rdbms::ConnPool &connPool);
virtual ~SchemaComparer();
virtual void compare() = 0;
virtual SchemaComparerResult compare() = 0;
std::string getCatalogueVersion();
protected:
cta::rdbms::Conn &m_conn;
cta::rdbms::Login &m_login;
cta::catalogue::Catalogue &m_catalogue;
const cta::rdbms::Login::DbType &m_dbType;
const std::string &m_schemaVersion;
cta::rdbms::ConnPool &m_catalogueConnPool;
std::unique_ptr<cta::catalogue::CatalogueMetadataGetter> m_catalogueMetadataGetter;
private:
virtual SchemaComparerResult compareTables() = 0;
virtual SchemaComparerResult compareIndexes() = 0;
};
}}
\ No newline at end of file
/**
* The CERN Tape Archive (CTA) project
* Copyright © 2018 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <iostream>
#include "SchemaComparerResult.hpp"
namespace cta {
namespace catalogue {
SchemaComparerResult::SchemaComparerResult():m_status(Status::SUCCESS) {
}
SchemaComparerResult::SchemaComparerResult(const SchemaComparerResult& orig) {
m_diffs = orig.m_diffs;
m_status = orig.m_status;
}
SchemaComparerResult SchemaComparerResult::operator=(const SchemaComparerResult &other){
if(this != &other){
m_diffs = other.m_diffs;
m_status = other.m_status;
}
return *this;
}
SchemaComparerResult SchemaComparerResult::operator +=(const SchemaComparerResult& other){
m_diffs.insert(m_diffs.end(),other.m_diffs.begin(),other.m_diffs.end());
if(m_status == Status::SUCCESS){
// The status should not change if it is failed
m_status = other.m_status;
}
return *this;
}
SchemaComparerResult::~SchemaComparerResult() {
}
void SchemaComparerResult::addDiff(const std::string& diff){
m_diffs.emplace_back(diff);
m_status = Status::FAILED;
}
SchemaComparerResult::Status SchemaComparerResult::getStatus() const {
return m_status;
}
void SchemaComparerResult::printDiffs() const {
for(auto &diff: m_diffs){
std::cout << diff << std::endl;
}
}
std::string SchemaComparerResult::StatusToString(const Status& status){
switch(status){
case Status::SUCCESS:
return "SUCCESS";
case Status::FAILED:
return "FAILED";
default:
return "UnknownStatus";
}
}
}}
\ No newline at end of file
/**
* The CERN Tape Archive (CTA) project
* Copyright © 2018 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <list>
#include <string>
namespace cta {
namespace catalogue {
class SchemaComparerResult {
public:
enum Status {
SUCCESS,
FAILED
};
static std::string StatusToString(const Status& status);
SchemaComparerResult();
SchemaComparerResult(const SchemaComparerResult& orig);
SchemaComparerResult operator=(const SchemaComparerResult &other);
SchemaComparerResult operator+=(const SchemaComparerResult &other);
void printDiffs() const;
Status getStatus() const;
void addDiff(const std::string &diff);
virtual ~SchemaComparerResult();
private:
std::list<std::string> m_diffs;
Status m_status;
};
}}
......@@ -30,13 +30,10 @@ SchemaCreatingSqliteCatalogue::SchemaCreatingSqliteCatalogue(
log::Logger &log,
const std::string &filename,
const uint64_t nbConns,
const uint64_t nbArchiveFileListingConns,
const bool createSchema):
const uint64_t nbArchiveFileListingConns):
SqliteCatalogue(log, filename, nbConns, nbArchiveFileListingConns) {
try {