Commit 362016ef authored by Cedric CAFFY's avatar Cedric CAFFY
Browse files

Created cta-statistics executable

Adapted SchemaChecker to be compatible with cta-statistics database
parent 0c53e15d
......@@ -150,9 +150,12 @@ ELSE(DEFINED PackageOnly)
add_subdirectory(scheduler)
add_subdirectory(tapeserver)
add_subdirectory(XRootdSSiRmcd)
add_subdirectory(statistics)
add_subdirectory(continuousintegration/orchestration/tests)
#Generate version information
configure_file(${PROJECT_SOURCE_DIR}/version.hpp.in
${CMAKE_BINARY_DIR}/version.h)
......
......@@ -289,7 +289,7 @@ set_property(TARGET cta-catalogue-admin-user-create APPEND PROPERTY INSTALL_RPAT
install(TARGETS cta-catalogue-admin-user-create DESTINATION /usr/bin)
install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/cta-catalogue-admin-user-create.1cta DESTINATION /usr/share/man/man1)
add_executable(cta-catalogue-schema-verify
set (SCHEMA_CHECKER_LIB_SRC_FILES
SQLiteSchemaInserter.cpp
SchemaSqlStatementsReader.cpp
SchemaComparer.cpp
......@@ -298,16 +298,22 @@ add_executable(cta-catalogue-schema-verify
SchemaComparerResult.cpp
SchemaChecker.cpp
CatalogueMetadataGetter.cpp
VerifySchemaCmd.cpp
VerifySchemaCmdLineArgs.cpp
VerifySchemaCmdMain.cpp
CatalogueSchema.cpp
OracleCatalogueSchema.cpp
SqliteCatalogueSchema.cpp
PostgresCatalogueSchema.cpp
MysqlCatalogueSchema.cpp)
MysqlCatalogueSchema.cpp
)
add_library (ctaschemachecker SHARED
${SCHEMA_CHECKER_LIB_SRC_FILES})
add_executable(cta-catalogue-schema-verify
VerifySchemaCmd.cpp
VerifySchemaCmdLineArgs.cpp
VerifySchemaCmdMain.cpp)
target_link_libraries(cta-catalogue-schema-verify ctacatalogue)
target_link_libraries(cta-catalogue-schema-verify ctacatalogue ctaschemachecker)
set_property(TARGET cta-catalogue-schema-verify APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH})
set_property(TARGET cta-catalogue-schema-verify APPEND PROPERTY INSTALL_RPATH ${ORACLE-INSTANTCLIENT_RPATH})
......
......@@ -59,6 +59,15 @@ SchemaComparerResult SQLiteSchemaComparer::compareTables(){
return res;
}
SchemaComparerResult SQLiteSchemaComparer::compareTablesInList(const std::list<std::string> tableNamesToCompare){
insertSchemaInSQLite();
std::list<std::string> catalogueTables = m_catalogueMetadataGetter.getTableNames();
std::list<std::string> schemaTables = m_schemaMetadataGetter->getTableNames();
SchemaComparerResult res = compareTables(catalogueTables,schemaTables);
//TODO use the list of table names to compare
return res;
}
void SQLiteSchemaComparer::insertSchemaInSQLite() {
if(!m_isSchemaInserted){
if(m_schemaSqlStatementsReader != nullptr){
......
......@@ -40,7 +40,8 @@ public:
SchemaComparerResult compareAll() override;
SchemaComparerResult compareIndexes() override;
SchemaComparerResult compareTables() override;
SchemaComparerResult compareTablesInList(const std::list<std::string> tableNamesToCompare) override;
virtual ~SQLiteSchemaComparer();
private:
......
......@@ -29,17 +29,6 @@ SchemaChecker::SchemaChecker(rdbms::Login::DbType dbType,cta::rdbms::Conn &conn)
SchemaChecker::~SchemaChecker() {
}
void SchemaChecker::useSQLiteSchemaComparer(const cta::optional<std::string> allSchemasVersionsDirectory){
m_schemaComparer.reset(new SQLiteSchemaComparer(*m_catalogueMetadataGetter));
std::unique_ptr<SchemaSqlStatementsReader> schemaSqlStatementsReader;
if(allSchemasVersionsDirectory){
schemaSqlStatementsReader.reset(new DirectoryVersionsSqlStatementsReader(m_dbType,m_catalogueMetadataGetter->getCatalogueVersion().getSchemaVersion<std::string>(),allSchemasVersionsDirectory.value()));
} else {
schemaSqlStatementsReader.reset(new MapSqlStatementsReader(m_dbType,m_catalogueMetadataGetter->getCatalogueVersion().getSchemaVersion<std::string>()));
}
m_schemaComparer->setSchemaSqlStatementsReader(std::move(schemaSqlStatementsReader));
}
SchemaChecker::Status SchemaChecker::compareSchema(){
if(m_schemaComparer == nullptr){
throw cta::exception::Exception("No schema comparer used. Please specify the schema comparer by using the methods useXXXXSchemaComparer()");
......@@ -81,4 +70,40 @@ void SchemaChecker::checkSchemaNotUpgrading(){
}
}
SchemaChecker::Builder::Builder(cta::rdbms::Login::DbType dbType, cta::rdbms::Conn& conn):m_dbType(dbType),m_catalogueConn(conn){
m_catalogueMetadataGetter.reset(CatalogueMetadataGetterFactory::create(m_dbType,m_catalogueConn));
}
SchemaChecker::Builder & SchemaChecker::Builder::useSQLiteSchemaComparer(){
m_schemaComparer.reset(new SQLiteSchemaComparer(*m_catalogueMetadataGetter));
return *this;
}
SchemaChecker::Builder& SchemaChecker::Builder::useDirectorySchemaReader(const std::string& allSchemasVersionsDirectory) {
m_schemaSqlStatementsReader.reset(new DirectoryVersionsSqlStatementsReader(m_dbType,m_catalogueMetadataGetter->getCatalogueVersion().getSchemaVersion<std::string>(),allSchemasVersionsDirectory));
return *this;
}
SchemaChecker::Builder& SchemaChecker::Builder::useMapStatementsReader() {
m_schemaSqlStatementsReader.reset(new MapSqlStatementsReader(m_dbType,m_catalogueMetadataGetter->getCatalogueVersion().getSchemaVersion<std::string>()));
return *this;
}
SchemaChecker::Builder& SchemaChecker::Builder::useStringStatementsReader() {
m_schemaSqlStatementsReader.reset(new StringSqlStatementsReader());
return *this;
}
std::unique_ptr<SchemaChecker> SchemaChecker::Builder::build() {
std::unique_ptr<SchemaChecker> schemaChecker(new SchemaChecker(m_dbType,m_catalogueConn));
if(m_schemaComparer != nullptr){
schemaChecker->m_schemaComparer = std::move(m_schemaComparer);
schemaChecker->m_schemaComparer->setSchemaSqlStatementsReader(std::move(m_schemaSqlStatementsReader));
return std::move(schemaChecker);
}
throw cta::exception::Exception("SchemaChecker::Builder::build(), a SchemaComparer should be set using the useXXXXSchemaComparer() method");
}
}}
\ No newline at end of file
......@@ -45,25 +45,11 @@ public:
OK,
FAILURE
};
/**
* Constructor of the SchemaChecker class
* @param dbType the type of the database to check against
* @param conn the connection of the database to check
*/
SchemaChecker(rdbms::Login::DbType dbType,cta::rdbms::Conn &conn);
/**
* Destructor
*/
virtual ~SchemaChecker();
/**
* Set the SQLiteSchemaComparer in order to run the schema comparison.
* The SQLiteSchemaComparer creates a InMemory SQLite database with the
* statements defined in the allSchemasDirectoryPath and will compare what is in SQLite with
* what is in the catalogue database.
*
* @param allSchemaVersionsDirectoryPath the schema sql statement reader used to get the schema
*/
void useSQLiteSchemaComparer(cta::optional<std::string> allSchemaVersionsDirectoryPath);
/**
* Compare the schema by using a SchemaComparer
* @throws Exception if no SchemaComparer has been set.
......@@ -81,7 +67,30 @@ public:
*/
void checkSchemaNotUpgrading();
class Builder {
public:
Builder(cta::rdbms::Login::DbType dbType, cta::rdbms::Conn &conn);
Builder & useSQLiteSchemaComparer();
Builder & useDirectorySchemaReader(const std::string &allSchemasVersionsDirectory);
Builder & useMapStatementsReader();
Builder & useStringStatementsReader();
std::unique_ptr<SchemaChecker> build();
private:
cta::rdbms::Login::DbType m_dbType;
cta::rdbms::Conn &m_catalogueConn;
std::unique_ptr<SchemaComparer> m_schemaComparer;
std::unique_ptr<CatalogueMetadataGetter> m_catalogueMetadataGetter;
std::unique_ptr<SchemaSqlStatementsReader> m_schemaSqlStatementsReader;
};
private:
/**
* Constructor of the SchemaChecker class
* @param dbType the type of the database to check against
* @param conn the connection of the database to check
*/
SchemaChecker(rdbms::Login::DbType dbType,cta::rdbms::Conn &conn);
/**
* Catalogue-to-check database type
*/
......
......@@ -62,6 +62,13 @@ public:
* @return a SchemaComparerResult that will contain the differences if there are some
*/
virtual SchemaComparerResult compareIndexes() = 0;
/**
* Compare only the tables in the list given in parameter
* @param tableNamesToCompare the tables to compare between the Schema and the database
* @return a SchemaComparerResult that will contain the differences if there are some
*/
virtual SchemaComparerResult compareTablesInList(const std::list<std::string> tableNamesToCompare) = 0;
/**
* Sets the way the schema sql statements will be read to do the schemas comparison
......
......@@ -34,6 +34,8 @@ namespace cta{ namespace catalogue{
//////////////////////////////////////////////////////////////////
// SchemaSqlStatementsReader
//////////////////////////////////////////////////////////////////
SchemaSqlStatementsReader::SchemaSqlStatementsReader(){}
SchemaSqlStatementsReader::SchemaSqlStatementsReader(const cta::rdbms::Login::DbType dbType):m_dbType(dbType) {
}
......@@ -170,5 +172,15 @@ std::list<std::string> MapSqlStatementsReader::getStatements(){
}
}
//////////////////////////////////////////////////////////////////
// StringSqlStatementsReader
//////////////////////////////////////////////////////////////////
StringSqlStatementsReader::StringSqlStatementsReader():SchemaSqlStatementsReader(){}
//TODO : Refactor
std::list<std::string> StringSqlStatementsReader::getStatements(){
return std::list<std::string>();
}
}}
......@@ -24,6 +24,7 @@ namespace cta{ namespace catalogue {
class SchemaSqlStatementsReader {
public:
SchemaSqlStatementsReader();
SchemaSqlStatementsReader(const cta::rdbms::Login::DbType dbType);
SchemaSqlStatementsReader(const SchemaSqlStatementsReader& orig);
virtual ~SchemaSqlStatementsReader();
......@@ -83,4 +84,10 @@ private:
std::string m_catalogueVersion;
};
class StringSqlStatementsReader: public SchemaSqlStatementsReader{
public:
StringSqlStatementsReader();
virtual std::list<std::string> getStatements();
};
}}
\ No newline at end of file
......@@ -74,12 +74,15 @@ int VerifySchemaCmd::exceptionThrowingMain(const int argc, char *const *const ar
return 1;
}
cta::catalogue::SchemaChecker schemaChecker(login.dbType,conn);
cta::optional<std::string> allSchemasDirectoryPathOpt;
schemaChecker.useSQLiteSchemaComparer(allSchemasDirectoryPathOpt);
SchemaChecker::Status comparisonStatus = schemaChecker.compareSchema();
schemaChecker.checkNoParallelTables();
schemaChecker.checkSchemaNotUpgrading();
SchemaChecker::Builder schemaCheckerBuilder(login.dbType,conn);
std::unique_ptr<SchemaChecker> schemaChecker(schemaCheckerBuilder
.useMapStatementsReader()
.useSQLiteSchemaComparer()
.build());
SchemaChecker::Status comparisonStatus = schemaChecker->compareSchema();
schemaChecker->checkNoParallelTables();
schemaChecker->checkSchemaNotUpgrading();
if(comparisonStatus == SchemaChecker::Status::FAILURE){
return 1;
}
......
# The CERN Tape Archive (CTA) project
# Copyright (C) 2015 CERN
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
cmake_minimum_required (VERSION 2.6)
find_package(Protobuf3 REQUIRED)
include_directories (${ORACLE-INSTANTCLIENT_INCLUDE_DIRS})
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wshadow")
set (STATISTICS_LIB_SRC_FILES
StatisticsCmd.cpp
StatisticsCmdLineArgs.cpp
StatisticsCmdMain.cpp
)
add_library (ctastatistics
${STATISTICS_LIB_SRC_FILES})
set_property(TARGET ctastatistics PROPERTY SOVERSION "${CTA_SOVERSION}")
set_property(TARGET ctastatistics PROPERTY VERSION "${CTA_LIBVERSION}")
install (TARGETS ctastatistics DESTINATION usr/${CMAKE_INSTALL_LIBDIR})
target_link_libraries (ctastatistics
ctacatalogue
ctacommon
ctaschemachecker
ctardbms)
install (FILES ${CMAKE_SOURCE_DIR}/catalogue/cta-catalogue.conf.example
DESTINATION ${CMAKE_INSTALL_SYSCONFDIR}/cta
PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ)
add_executable(cta-statistics
StatisticsCmd.cpp
StatisticsCmdLineArgs.cpp
StatisticsCmdMain.cpp
)
target_link_libraries (cta-statistics ctacatalogue ctaschemachecker)
set_property(TARGET cta-statistics APPEND PROPERTY INSTALL_RPATH ${ORACLE-INSTANTCLIENT_RPATH})
install (TARGETS cta-statistics DESTINATION /usr/bin)
install (FILES ${CMAKE_CURRENT_SOURCE_DIR}/cta-statistics.1cta DESTINATION /usr/share/man/man1)
set (STATISTICS_CMD_LINE_UNIT_TESTS_LIB_SRC_FILES
StatisticsCmdLineArgsTest.cpp
)
add_library (ctastatisticscmdlineunittests SHARED
${STATISTICS_CMD_LINE_UNIT_TESTS_LIB_SRC_FILES})
set_property(TARGET ctastatisticscmdlineunittests PROPERTY SOVERSION "${CTA_SOVERSION}")
set_property(TARGET ctastatisticscmdlineunittests PROPERTY VERSION "${CTA_LIBVERSION}")
install (TARGETS ctastatisticscmdlineunittests DESTINATION usr/${CMAKE_INSTALL_LIBDIR})
/*
* The CERN Tape Archive (CTA) project
* Copyright (C) 2019 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "rdbms/ConnPool.hpp"
#include "rdbms/AutocommitMode.hpp"
#include "StatisticsCmd.hpp"
#include "catalogue/SchemaChecker.hpp"
namespace cta {
namespace statistics {
//------------------------------------------------------------------------------
// constructor
//------------------------------------------------------------------------------
StatisticsCmd::StatisticsCmd(std::istream &inStream, std::ostream &outStream, std::ostream &errStream):
CmdLineTool(inStream, outStream, errStream) {
}
//------------------------------------------------------------------------------
// destructor
//------------------------------------------------------------------------------
StatisticsCmd::~StatisticsCmd() noexcept {
}
//------------------------------------------------------------------------------
// exceptionThrowingMain
//------------------------------------------------------------------------------
int StatisticsCmd::exceptionThrowingMain(const int argc, char *const *const argv) {
const StatisticsCmdLineArgs cmdLineArgs(argc, argv);
if(cmdLineArgs.help) {
printUsage(m_out);
return 0;
}
std::cout << cmdLineArgs.catalogueDbConfigPath << std::endl;
std::cout << cmdLineArgs.statisticsDbConfigPath << std::endl;
const uint64_t maxNbConns = 1;
auto loginCatalogue = rdbms::Login::parseFile(cmdLineArgs.catalogueDbConfigPath);
rdbms::ConnPool catalogueConnPool(loginCatalogue, maxNbConns);
auto catalogueConn = catalogueConnPool.getConn();
auto loginStatistics = rdbms::Login::parseFile(cmdLineArgs.statisticsDbConfigPath);
rdbms::ConnPool statisticsConnPool(loginStatistics, maxNbConns);
auto statisticsConn = statisticsConnPool.getConn();
/*cta::catalogue::SchemaChecker catalogueSchemaChecker(loginCatalogue.dbType,catalogueConn);
cta::catalogue::SchemaChecker statisticsSchemaChecker(loginStatistics.dbType,statisticsConn);*/
// catalogueSchemaChecker.useSQLiteSchemaComparer();
return 0;
}
//------------------------------------------------------------------------------
// tableExists
//------------------------------------------------------------------------------
bool StatisticsCmd::tableExists(const std::string tableName, rdbms::Conn &conn) const {
const auto names = conn.getTableNames();
for(const auto &name : names) {
if(tableName == name) {
return true;
}
}
return false;
}
//------------------------------------------------------------------------------
// printUsage
//------------------------------------------------------------------------------
void StatisticsCmd::printUsage(std::ostream &os) {
StatisticsCmdLineArgs::printUsage(os);
}
} // namespace catalogue
} // namespace cta
/*
* The CERN Tape Archive (CTA) project
* Copyright (C) 2015 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "catalogue/CmdLineTool.hpp"
#include "catalogue/CatalogueSchema.hpp"
#include "rdbms/Conn.hpp"
#include "rdbms/Login.hpp"
#include "StatisticsCmdLineArgs.hpp"
namespace cta {
namespace statistics {
/**
* Command-line tool for verifying the catalogue schema.
*/
class StatisticsCmd: public cta::catalogue::CmdLineTool {
public:
/**
* Constructor.
*
* @param inStream Standard input stream.
* @param outStream Standard output stream.
* @param errStream Standard error stream.
*/
StatisticsCmd(std::istream &inStream, std::ostream &outStream, std::ostream &errStream);
/**
* Destructor.
*/
~StatisticsCmd() noexcept;
enum class VerifyStatus { OK, INFO, ERROR, UNKNOWN };
private:
/**
* An exception throwing version of main().
*
* @param argc The number of command-line arguments including the program name.
* @param argv The command-line arguments.
* @return The exit value of the program.
*/
int exceptionThrowingMain(const int argc, char *const *const argv) override;
/**
* Prints the usage message of the command-line tool.
*
* @param os The output stream to which the usage message is to be printed.
*/
void printUsage(std::ostream &os) override;
/**
* Returns true if the table with the specified name exists in the database
* schema of the specified database connection.
*
* @param tableName The name of the database table.
* @param conn The database connection.
* @return True if the table exists.
*/
bool tableExists(const std::string tableName, rdbms::Conn &conn) const;
}; // class VerifySchemaCmd
} // namespace statistics
} // namespace cta
/*
* The CERN Tape Archive (CTA) project
* Copyright (C) 2015 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "StatisticsCmdLineArgs.hpp"
#include "common/exception/CommandLineNotParsed.hpp"
#include <getopt.h>
#include <ostream>
#include <iostream>
namespace cta {
namespace statistics {
//------------------------------------------------------------------------------
// constructor
//------------------------------------------------------------------------------
StatisticsCmdLineArgs::StatisticsCmdLineArgs(const int argc, char *const *const argv):
help(false) {
static struct option longopts[] = {
{"catalogueconf",required_argument,NULL,'c'},
{"statisticsconf",required_argument,NULL,'s'},
{"help", no_argument, NULL, 'h'},
{NULL , 0, NULL, 0}
};
// Prevent getopt() from printing an error message if it does not recognize
// an option character
opterr = 0;
int opt = 0;
while((opt = getopt_long(argc, argv, ":hc:s:", longopts, NULL)) != -1) {
switch(opt) {
case 'h':
help = true;
break;
case 'c':
catalogueDbConfigPath = optarg;
break;
case 's':
statisticsDbConfigPath = optarg;
break;
case ':': // Missing parameter
{
exception::CommandLineNotParsed ex;
ex.getMessage() << "The -" << (char)optopt << " option requires a parameter";
throw ex;
}
case '?': // Unknown option
{
exception::CommandLineNotParsed ex;
if(0 == optopt) {
ex.getMessage() << "Unknown command-line option";
} else {
ex.getMessage() << "Unknown command-line option: -" << (char)optopt;
}
throw ex;
}
default:
{
exception::CommandLineNotParsed ex;
ex.getMessage() <<
"getopt_long returned the following unknown value: 0x" <<
std::hex << (int)opt;
throw ex;
}
} // switch(opt)
} // while getopt_long()
// There is no need to continue parsing when the help option is set
if(help) {
return;
}