Commit 15aafada authored by Cedric CAFFY's avatar Cedric CAFFY
Browse files

Inserted the xxx_catalogue_schema.sql in a sqlite database

parent 0abecb9a
......@@ -267,7 +267,7 @@ CREATE TABLE ACTIVITIES_WEIGHTS (
LAST_UPDATE_TIME BIGINT UNSIGNED CONSTRAINT ACTIV_WEIGHTS_LUT_NN NOT NULL
);
CREATE TABLE USAGESTATS (
GID BIGINT UNSIGNED DEFAULT 0 CONSTRAINT NN_USAGESTATS_GID NOT NULL,
GID INT UNSIGNED DEFAULT 0 CONSTRAINT NN_USAGESTATS_GID NOT NULL,
TIMESTAMP BIGINT UNSIGNED DEFAULT 0 CONSTRAINT NN_USAGESTATS_TS NOT NULL,
MAXFILEID BIGINT UNSIGNED,
FILECOUNT BIGINT UNSIGNED,
......@@ -280,7 +280,7 @@ CREATE TABLE USAGESTATS (
);
CREATE TABLE EXPERIMENTS (
NAME VARCHAR(20),
GID BIGINT UNSIGNED CONSTRAINT EXPERIMENTS_GID_PK PRIMARY KEY
GID INT UNSIGNED CONSTRAINT EXPERIMENTS_GID_PK PRIMARY KEY
);
INSERT INTO CTA_CATALOGUE(
SCHEMA_VERSION_MAJOR,
......
......@@ -289,7 +289,7 @@ CREATE TABLE ACTIVITIES_WEIGHTS (
LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT ACTIV_WEIGHTS_LUT_NN NOT NULL
);
CREATE TABLE USAGESTATS (
GID NUMERIC(6, 0) DEFAULT 0 CONSTRAINT NN_USAGESTATS_GID NOT NULL,
GID NUMERIC(10, 0) DEFAULT 0 CONSTRAINT NN_USAGESTATS_GID NOT NULL,
TIMESTAMP NUMERIC(20, 0) DEFAULT 0 CONSTRAINT NN_USAGESTATS_TS NOT NULL,
MAXFILEID NUMERIC(20, 0),
FILECOUNT NUMERIC(20, 0),
......@@ -302,7 +302,7 @@ CREATE TABLE USAGESTATS (
);
CREATE TABLE EXPERIMENTS (
NAME VARCHAR2(20),
GID NUMERIC(6, 0) CONSTRAINT EXPERIMENTS_GID_PK PRIMARY KEY
GID NUMERIC(10, 0) CONSTRAINT EXPERIMENTS_GID_PK PRIMARY KEY
);
INSERT INTO CTA_CATALOGUE(
SCHEMA_VERSION_MAJOR,
......
......@@ -274,7 +274,7 @@ CREATE TABLE ACTIVITIES_WEIGHTS (
LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT ACTIV_WEIGHTS_LUT_NN NOT NULL
);
CREATE TABLE USAGESTATS (
GID NUMERIC(6, 0) DEFAULT 0 CONSTRAINT NN_USAGESTATS_GID NOT NULL,
GID NUMERIC(10, 0) DEFAULT 0 CONSTRAINT NN_USAGESTATS_GID NOT NULL,
TIMESTAMP NUMERIC(20, 0) DEFAULT 0 CONSTRAINT NN_USAGESTATS_TS NOT NULL,
MAXFILEID NUMERIC(20, 0),
FILECOUNT NUMERIC(20, 0),
......@@ -287,7 +287,7 @@ CREATE TABLE USAGESTATS (
);
CREATE TABLE EXPERIMENTS (
NAME VARCHAR(20),
GID NUMERIC(6, 0) CONSTRAINT EXPERIMENTS_GID_PK PRIMARY KEY
GID NUMERIC(10, 0) CONSTRAINT EXPERIMENTS_GID_PK PRIMARY KEY
);
INSERT INTO CTA_CATALOGUE(
SCHEMA_VERSION_MAJOR,
......
......@@ -285,6 +285,10 @@ install(TARGETS cta-catalogue-admin-user-create DESTINATION /usr/bin)
install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/cta-catalogue-admin-user-create.1cta DESTINATION /usr/share/man/man1)
add_executable(cta-catalogue-schema-verify
SQLiteSchemaInserter.cpp
SchemaComparer.cpp
SQLiteSchemaComparer.cpp
DbToSQLiteStatementTransformer.cpp
VerifySchemaCmd.cpp
VerifySchemaCmdLineArgs.cpp
VerifySchemaCmdMain.cpp
......
/**
* The CERN Tape Archive (CTA) project
* Copyright © 2018 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <iostream>
#include "DbToSQLiteStatementTransformer.hpp"
#include "common/utils/Regex.hpp"
#include "common/utils/utils.hpp"
namespace cta {
namespace catalogue {
/**********************************/
/* DbToSQLiteStatementTransformer */
/**********************************/
DbToSQLiteStatementTransformer::DbToSQLiteStatementTransformer(const std::string &statement):m_statement(statement) {
}
DbToSQLiteStatementTransformer::~DbToSQLiteStatementTransformer() {
}
std::string DbToSQLiteStatementTransformer::transform(){
return m_statement;
}
CreateGlobalTempTableToSQLiteStatementTransformer::CreateGlobalTempTableToSQLiteStatementTransformer(const std::string &statement):DbToSQLiteStatementTransformer(statement){}
std::string CreateGlobalTempTableToSQLiteStatementTransformer::transform(){
utils::searchAndReplace(m_statement,"GLOBAL TEMPORARY ","");
utils::searchAndReplace(m_statement,"ON COMMIT DELETE ROWS;",";");
return m_statement;
}
DeleteStatementTransformer::DeleteStatementTransformer(const std::string& statement):DbToSQLiteStatementTransformer(statement){}
std::string DeleteStatementTransformer::transform(){
return "";
}
std::unique_ptr<DbToSQLiteStatementTransformer> DbToSQLiteStatementTransformerFactory::create(const std::string &statement){
typedef DbToSQLiteStatementTransformerFactory::StatementType StatementType;
StatementType stmtType = statementToStatementType(statement);
std::unique_ptr<DbToSQLiteStatementTransformer> ret;
switch(stmtType){
case StatementType::CREATE_TABLE:
case StatementType::CREATE_INDEX:
case StatementType::INSERT_INTO_CTA_VERSION:
ret.reset(new DbToSQLiteStatementTransformer(statement));
break;
case StatementType::CREATE_GLOBAL_TEMPORARY_TABLE:
ret.reset(new CreateGlobalTempTableToSQLiteStatementTransformer(statement));
break;
case StatementType::CREATE_SEQUENCE:
case StatementType::SKIP:
ret.reset(new DeleteStatementTransformer(statement));
break;
default:
ret.reset(new DbToSQLiteStatementTransformer(statement));
break;
}
return ret;
}
const std::map<std::string,DbToSQLiteStatementTransformerFactory::StatementType> DbToSQLiteStatementTransformerFactory::regexToStatementMap = DbToSQLiteStatementTransformerFactory::initializeRegexToStatementMap();
DbToSQLiteStatementTransformerFactory::StatementType DbToSQLiteStatementTransformerFactory::statementToStatementType(const std::string &statement){
for(auto &kv: regexToStatementMap){
utils::Regex regexToTest(kv.first);
if(regexToTest.exec(statement).size() != 0){
return kv.second;
}
}
return StatementType::SKIP;
}
std::map<std::string,DbToSQLiteStatementTransformerFactory::StatementType> DbToSQLiteStatementTransformerFactory::initializeRegexToStatementMap()
{
typedef DbToSQLiteStatementTransformerFactory::StatementType StatementType;
std::map<std::string,StatementType> ret;
ret["CREATE TABLE ([a-zA-Z_]+)"] = StatementType::CREATE_TABLE;
ret["CREATE INDEX ([a-zA-Z_]+)"] = StatementType::CREATE_INDEX;
ret["CREATE GLOBAL TEMPORARY TABLE ([a-zA-Z_]+)"] = StatementType::CREATE_GLOBAL_TEMPORARY_TABLE;
ret["CREATE SEQUENCE ([a-zA-Z_]+)"] = StatementType::CREATE_SEQUENCE;
ret["INSERT INTO CTA_CATALOGUE([a-zA-Z_]+)"] = StatementType::INSERT_INTO_CTA_VERSION;
ret["INSERT INTO ([a-zA-Z_]+)"] = StatementType::SKIP;
return ret;
}
}}
\ No newline at end of file
/**
* The CERN Tape Archive (CTA) project
* Copyright © 2018 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <string>
#include <map>
#include <memory>
#pragma once
namespace cta {
namespace catalogue {
class DbToSQLiteStatementTransformer {
public:
DbToSQLiteStatementTransformer(const std::string &statement);
virtual ~DbToSQLiteStatementTransformer();
virtual std::string transform();
protected:
std::string m_statement;
};
class CreateGlobalTempTableToSQLiteStatementTransformer: public DbToSQLiteStatementTransformer {
public:
CreateGlobalTempTableToSQLiteStatementTransformer(const std::string &statement);
std::string transform() override;
};
class DeleteStatementTransformer :public DbToSQLiteStatementTransformer {
public:
DeleteStatementTransformer(const std::string &statement);
std::string transform() override;
};
class DbToSQLiteStatementTransformerFactory {
enum StatementType {
CREATE_TABLE,
CREATE_INDEX,
CREATE_GLOBAL_TEMPORARY_TABLE,
CREATE_SEQUENCE,
INSERT_INTO_CTA_VERSION,
SKIP
};
private:
static const std::map<std::string,StatementType> regexToStatementMap;
static std::map<std::string,StatementType> initializeRegexToStatementMap();
static StatementType statementToStatementType(const std::string &statement);
public:
static std::unique_ptr<DbToSQLiteStatementTransformer> create(const std::string &statement);
};
}}
\ No newline at end of file
/**
* The CERN Tape Archive (CTA) project
* Copyright © 2018 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
/*
* File: SQLiteSchemaComparer.cpp
* Author: cedric
*
* Created on December 10, 2019, 11:34 AM
*/
#include "SQLiteSchemaComparer.hpp"
#include "SQLiteSchemaInserter.hpp"
namespace cta {
namespace catalogue {
SQLiteSchemaComparer::SQLiteSchemaComparer(cta::rdbms::Conn &connection, cta::rdbms::Login &login, cta::catalogue::Catalogue& catalogue): SchemaComparer(connection,login,catalogue) {
}
SQLiteSchemaComparer::~SQLiteSchemaComparer() {
}
void SQLiteSchemaComparer::compare(){
insertSchemaInSQLite();
//compare
//return the results
}
void SQLiteSchemaComparer::insertSchemaInSQLite() {
std::map<std::string, uint64_t> schemaVersion = m_catalogue.getSchemaVersion();
uint64_t schemaVersionMajor = schemaVersion.at("SCHEMA_VERSION_MAJOR");
uint64_t schemaVersionMinor = schemaVersion.at("SCHEMA_VERSION_MINOR");
std::string schemaVersionStr = std::to_string(schemaVersionMajor)+"."+std::to_string(schemaVersionMinor);
cta::catalogue::SQLiteSchemaInserter schemaInserter(schemaVersionStr,m_login.dbType,"/home/cedric/CTA/catalogue/","/home/cedric/CTA-execute/sqliteDbFile");
schemaInserter.insert();
}
}}
\ No newline at end of file
/**
* The CERN Tape Archive (CTA) project
* Copyright © 2018 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "SchemaComparer.hpp"
#pragma once
namespace cta {
namespace catalogue {
class SQLiteSchemaComparer: public SchemaComparer {
public:
SQLiteSchemaComparer(cta::rdbms::Conn &connection, cta::rdbms::Login &login, cta::catalogue::Catalogue& catalogue);
void compare() override;
virtual ~SQLiteSchemaComparer();
private:
void insertSchemaInSQLite();
};
}}
\ No newline at end of file
/**
* The CERN Tape Archive (CTA) project
* Copyright © 2018 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "SQLiteSchemaInserter.hpp"
#include "common/exception/Exception.hpp"
#include "common/utils/utils.hpp"
#include "DbToSQLiteStatementTransformer.hpp"
#include "common/log/DummyLogger.hpp"
#include <iostream>
#include <fstream>
namespace cta {
namespace catalogue {
SQLiteSchemaInserter::SQLiteSchemaInserter(const std::string & schemaVersion, const cta::rdbms::Login::DbType &dbType, const std::string &allVersionsSchemaDirectory,const std::string &sqliteFileName):m_schemaVersion(schemaVersion),m_dbType(dbType),m_allVersionSchemaDirectory(allVersionsSchemaDirectory),m_sqliteFileName(sqliteFileName) {
log::DummyLogger dl("dummy","dummy");
m_sqliteCatalogue = new SchemaCreatingSqliteCatalogue(dl,m_sqliteFileName,1,1,false);
m_conn = m_sqliteCatalogue->m_connPool.getConn();
}
SQLiteSchemaInserter::~SQLiteSchemaInserter() {
if(m_sqliteCatalogue != nullptr) {
delete m_sqliteCatalogue;
}
::remove(m_sqliteFileName.c_str());
}
void SQLiteSchemaInserter::insert() {
std::list<std::string> statements = getAllStatementsFromSchema(readSchemaFromFile());
std::string transformedSchema;
for(auto& stmt: statements){
transformedSchema.append(DbToSQLiteStatementTransformerFactory::create(stmt)->transform());
}
m_sqliteCatalogue->executeNonQueries(m_conn,transformedSchema);
}
void SQLiteSchemaInserter::executeStatements(const std::string &sqliteStatements){
}
std::list<std::string> SQLiteSchemaInserter::getAllStatementsFromSchema(const std::string &schema){
std::list<std::string> statements;
std::string::size_type searchPos = 0;
std::string::size_type findResult = std::string::npos;
try {
while(std::string::npos != (findResult = schema.find(';', searchPos))) {
// Calculate the length of the current statement without the trailing ';'
const std::string::size_type stmtLen = findResult - searchPos;
const std::string sqlStmt = utils::trimString(schema.substr(searchPos, stmtLen));
searchPos = findResult + 1;
if(0 < sqlStmt.size()) { // Ignore empty statements
statements.push_back(sqlStmt+";");
}
}
} catch(exception::Exception &ex) {
throw exception::Exception(std::string(__FUNCTION__) + " failed: " + ex.getMessage().str());
}
return statements;
}
std::string SQLiteSchemaInserter::readSchemaFromFile() {
std::string schemaFilePath = getSchemaFilePath();
std::ifstream schemaFile(schemaFilePath);
if(schemaFile.fail()){
throw cta::exception::Exception("In SQLiteSchemaInserter::readSchemaFromFile, unable to open the file located in "+schemaFilePath);
}
std::string content((std::istreambuf_iterator<char>(schemaFile)),(std::istreambuf_iterator<char>()));
return content;
}
std::string SQLiteSchemaInserter::getSchemaFilePath() {
return m_allVersionSchemaDirectory+m_schemaVersion+"/"+getDatabaseType()+c_catalogueFileNameTrailer;
}
std::string SQLiteSchemaInserter::getDatabaseType() {
switch(m_dbType){
case rdbms::Login::DBTYPE_IN_MEMORY:
case rdbms::Login::DBTYPE_SQLITE:
return "sqlite";
case rdbms::Login::DBTYPE_POSTGRESQL:
return "postgres";
case rdbms::Login::DBTYPE_MYSQL:
return "mysql";
case rdbms::Login::DBTYPE_ORACLE:
return "oracle";
case rdbms::Login::DBTYPE_NONE:
throw exception::Exception("The database type should not be DBTYPE_NONE");
default:
{
exception::Exception ex;
ex.getMessage() << "Unknown database type: value=" << m_dbType;
throw ex;
}
}
}
}}
\ No newline at end of file
/**
* The CERN Tape Archive (CTA) project
* Copyright © 2018 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "rdbms/Login.hpp"
#include "SchemaCreatingSqliteCatalogue.hpp"
namespace cta {
namespace catalogue {
class SQLiteSchemaInserter {
public:
SQLiteSchemaInserter(const std::string & schemaVersion, const cta::rdbms::Login::DbType &dbType, const std::string &allVersionsSchemaDirectory,const std::string &sqliteFileName);
void insert();
virtual ~SQLiteSchemaInserter();
private:
std::string m_schemaVersion;
cta::rdbms::Login::DbType m_dbType;
std::string m_allVersionSchemaDirectory;
const std::string c_catalogueFileNameTrailer = "_catalogue_schema.sql";
std::string m_sqliteFileName;
cta::catalogue::SchemaCreatingSqliteCatalogue * m_sqliteCatalogue;
cta::rdbms::Conn m_conn;
std::string readSchemaFromFile();
std::list<std::string> getAllStatementsFromSchema(const std::string &schema);
std::string getDatabaseType();
std::string getSchemaFilePath();
void executeStatements(const std::string &statements);
};
}}
/**
* The CERN Tape Archive (CTA) project
* Copyright © 2018 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
/*
* File: SchemaComparer.cpp
* Author: cedric
*
* Created on December 10, 2019, 10:58 AM
*/
#include "SchemaComparer.hpp"
namespace cta {
namespace catalogue {
SchemaComparer::SchemaComparer(cta::rdbms::Conn &connection, cta::rdbms::Login &login, cta::catalogue::Catalogue& catalogue):m_conn(connection), m_login(login), m_catalogue(catalogue){
}
SchemaComparer::~SchemaComparer() {
}
}}
/**
* The CERN Tape Archive (CTA) project
* Copyright © 2018 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
/*
* File: SchemaComparer.hpp
* Author: cedric
*
* Created on December 10, 2019, 10:58 AM
*/
#pragma once
#include "rdbms/Conn.hpp"
#include "rdbms/Login.hpp"
#include "catalogue/Catalogue.hpp"
namespace cta {
namespace catalogue {
class SchemaComparer {
public:
SchemaComparer(cta::rdbms::Conn &connection, cta::rdbms::Login &login, cta::catalogue::Catalogue& catalogue);
virtual ~SchemaComparer();
virtual void compare() = 0;
protected:
cta::rdbms::Conn &m_conn;
cta::rdbms::Login &m_login;
cta::catalogue::Catalogue &m_catalogue;
};
}}
\ No newline at end of file
......@@ -30,10 +30,13 @@ SchemaCreatingSqliteCatalogue::SchemaCreatingSqliteCatalogue(
log::Logger &log,
const std::string &filename,
const uint64_t nbConns,
const uint64_t nbArchiveFileListingConns):
const uint64_t nbArchiveFileListingConns,
const bool createSchema):
SqliteCatalogue(log, filename, nbConns, nbArchiveFileListingConns) {
try {
createCatalogueSchema();
if(createSchema) {
createCatalogueSchema();
}
} catch(exception::Exception &ex) {
throw exception::Exception(std::string(__FUNCTION__) + " failed: " + ex.getMessage().str());
}
......
......@@ -28,7 +28,10 @@ class CatalogueFactory;
/**
* Asbtract CTA catalogue class to be used for unit testing.
*/
class SQLiteSchemaInserter;
class SchemaCreatingSqliteCatalogue: public SqliteCatalogue {
friend class SQLiteSchemaInserter;
public:
/**
......@@ -47,7 +50,8 @@ public:
log::Logger &log,
const std::string &filename,