Commit fac48f51 authored by Michael Davis's avatar Michael Davis
Browse files

[migration] Initial version of UpgradeDB tool

parent 100c9709
......@@ -147,7 +147,9 @@ ELSE(DEFINED PackageOnly)
add_subdirectory(scheduler)
add_subdirectory(tapeserver)
add_subdirectory(XRootdSSiRmcd)
add_subdirectory(migration/CTA)
#Generate version information
configure_file(${PROJECT_SOURCE_DIR}/version.hpp.in
${CMAKE_BINARY_DIR}/version.h)
......
# The CERN Tape Archive (CTA) project
# Copyright 2019 CERN
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
cmake_minimum_required (VERSION 2.6)
include_directories(${XRD_SSI_PB_DIR}/include ${XROOTD_INCLUDE_DIR} ${XROOTD_INCLUDE_DIR}/private)
# Upgrade DB tool
add_executable(upgrade-db UpgradeDB.cpp)
target_link_libraries(upgrade-db ctacatalogue)
#install(TARGETS upgrade-db DESTINATION usr/bin)
/*!
* @project The CERN Tape Archive (CTA)
* @brief Access Oracle DB for migration operations
* @copyright Copyright 2019 CERN
* @license This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <memory>
#include <rdbms/Login.hpp>
#include <rdbms/ConnPool.hpp>
namespace cta {
namespace migration {
/*!
* Manage a single database query over a single database connection
*/
class OracleDbConn
{
public:
OracleDbConn() : m_queryIsEmpty(true) {}
void connect(const std::string &dbconn = "", unsigned int max_num_conns = 1) {
// Initialise the connection pool
if(!dbconn.empty()) {
m_connPool.reset(new rdbms::ConnPool(rdbms::Login::parseString(dbconn), max_num_conns));
}
// Initialise this connection
m_conn = m_connPool->getConn();
}
void query(const std::string &sqlString) {
auto sql = m_conn.createStmt(sqlString);
m_rSet = sql.executeQuery();
m_queryIsEmpty = !m_rSet.next();
}
bool nextRow() {
return m_rSet.next();
}
std::string getResultColumnString(const std::string &col) const {
return m_rSet.columnString(col);
}
bool isQueryEmpty() const {
return m_queryIsEmpty;
}
private:
static std::unique_ptr<rdbms::ConnPool> m_connPool; //!< The pool of connections to the database
rdbms::Conn m_conn; //!< The connection we are using
rdbms::Rset m_rSet; //!< Result set for the last query executed
bool m_queryIsEmpty; //!< Track whether the last query had an empty result set
};
}} // namespace cta::migration
/*!
* @project The CERN Tape Archive (CTA)
* @brief Migration tool to upgrade Oracle DB
* @copyright Copyright 2019 CERN
* @license This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <iostream>
#include <sstream>
#include <vector>
#include <XrdSsiPbConfig.hpp>
#include "OracleDbConn.hpp"
namespace cta {
namespace migration {
//! DB Connection Pool
std::unique_ptr<rdbms::ConnPool> OracleDbConn::m_connPool;
class UpgradeDB
{
public:
UpgradeDB(const std::string &configfile);
bool processBatch();
private:
bool getNextBatch(std::vector<std::string> &paths);
OracleDbConn m_ctadb; //!< Oracle database for CTA Catalogue
unsigned int m_max_depth; //!< Maximum directory tree depth to import
unsigned int m_cur_depth; //!< Current directory tree depth
unsigned int m_batch_size; //!< Number of records to fetch from the DB at a time
};
UpgradeDB::UpgradeDB(const std::string &configfile)
{
// Parse configuration file
XrdSsiPb::Config config(configfile);
auto dbconn = config.getOptionValueStr("cta.db_login");
auto max_num_conns = config.getOptionValueInt("cta.max_num_connections");
auto max_depth = config.getOptionValueInt("cta.max_depth");
auto batch_size = config.getOptionValueInt("cta.batch_size");
// Connect to Oracle
if(!dbconn.first) {
throw std::runtime_error("castor.db_login must be specified in the config file in the form oracle:user/password@TNS");
}
m_ctadb.connect(dbconn.second, max_num_conns.first ? max_num_conns.second : 1);
// Set parameters and defaults
m_max_depth = max_depth.first ? max_depth.second : std::numeric_limits<unsigned int>::max();;
m_batch_size = batch_size.first ? batch_size.second : 1000;
}
bool UpgradeDB::getNextBatch(std::vector<std::string> &paths)
{
#if 0
for(unsigned int b = 0; b < m_batch_size; ++b) {
if(m_ctadb.isQueryEmpty() || !m_ctadb.nextRow()) {
if(m_cur_depth > m_max_depth) return false;
m_ctadb.query("select name from mdavis_cns_dirnames where depth=" + std::to_string(m_cur_depth++));
if(m_ctadb.isQueryEmpty()) return false;
}
paths.push_back(m_prefix + m_ctadb.getResultColumnString("NAME"));
++m_total_dirs;
}
#endif
return true;
}
bool UpgradeDB::processBatch()
{
#if 0
std::vector<std::string> paths;
bool isFinished = getNextBatch(paths);
if(!paths.empty()) {
int retc = m_eosgrpc->ContainerInsert(paths);
if(retc != 0) {
throw std::runtime_error("UpgradeDB::processBatch(): ContainerInsert failed with error " +
std::to_string(retc));
}
auto elapsed_time = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::steady_clock::now() - m_start_time);
std::cout << "Processed " << m_total_dirs << " directories in " << elapsed_time.count() << "s" << std::endl;
}
return isFinished;
#endif
return true;
}
}} // namespace cta::migration
void throwUsage(const std::string &program, const std::string &error_txt)
{
std::stringstream help;
help << program << ": " << error_txt << std::endl
<< "Usage: " << program << " [--config <config_file>]";
throw std::runtime_error(help.str());
}
int main(int argc, const char* argv[])
{
std::string configfile = "/home/ctadev/CTA/migration/castor-migration.conf";
try {
for(auto i = 1; i < argc; ++i) {
std::string option(argv[i]);
if(option == "--config" && argc > ++i) {
configfile = argv[i];
continue;
}
throwUsage(argv[0], "invalid option " + option);
}
cta::migration::UpgradeDB importDirs(configfile);
while(importDirs.processBatch()) ;
} catch(std::runtime_error &ex) {
std::cerr << ex.what() << std::endl;
return -1;
}
return 0;
}
# The CERN Tape Archive (CTA) project
# Copyright 2019 CERN
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
cmake_minimum_required (VERSION 2.6)
add_subdirectory(gephi)
#add_subdirectory(not_on_tape)
include_directories(${XRD_SSI_PB_DIR}/include ${XROOTD_INCLUDE_DIR} ${XROOTD_INCLUDE_DIR}/private
${CMAKE_BINARY_DIR}/eos_cta
${PROTOBUF3_INCLUDE_DIRS})
# Directory import tool
add_executable(eos-import-dirs EosImportDirs.cpp GrpcClient.cpp)
target_link_libraries(eos-import-dirs EosMigration ${PROTOBUF3_LIBRARIES} ${GRPC_LIBRARY} ${GRPC_GRPC++_LIBRARY} ctacatalogue)
set_property(TARGET eos-import-dirs APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH})
#install(TARGETS eos-import-dirs DESTINATION usr/bin)
#install(FILES eos-import-dirs.1cta DESTINATION usr/share/man/man1)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment