Commit 629b0956 authored by Michael Davis's avatar Michael Davis
Browse files

[migration] Separates DB migration tools from DB upgrade tools

- CASTOR migration tools stay under CTA/migration
- upgrade tools are now under CTA/upgrade_db
- adds FindGRPC.cmake (required for migration tools)
- updates submodule to v0.50 (required for EOS gRPC API)
parent eafd8175
......@@ -148,7 +148,8 @@ ELSE(DEFINED PackageOnly)
add_subdirectory(tapeserver)
add_subdirectory(XRootdSSiRmcd)
add_subdirectory(migration/CTA)
add_subdirectory(upgrade_db)
add_subdirectory(migration/gRPC)
#Generate version information
configure_file(${PROJECT_SOURCE_DIR}/version.hpp.in
......
#
# Locate and configure the gRPC library
#
# Adds the following targets:
#
# gRPC::grpc - gRPC library
# gRPC::grpc++ - gRPC C++ library
# gRPC::grpc++_reflection - gRPC C++ reflection library
# gRPC::grpc_cpp_plugin - C++ generator plugin for Protocol Buffers
#
#
# Generates C++ sources from the .proto files
#
# grpc_generate_cpp (<SRCS> <HDRS> <DEST> [<ARGN>...])
#
# SRCS - variable to define with autogenerated source files
# HDRS - variable to define with autogenerated header files
# DEST - directory where the source files will be created
# ARGN - .proto files
#
function(GRPC_GENERATE_CPP SRCS HDRS DEST)
if(NOT ARGN)
message(SEND_ERROR "Error: GRPC_GENERATE_CPP() called without any proto files")
return()
endif()
if(GRPC_GENERATE_CPP_APPEND_PATH)
# Create an include path for each file specified
foreach(FIL ${ARGN})
get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
get_filename_component(ABS_PATH ${ABS_FIL} PATH)
list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)
if(${_contains_already} EQUAL -1)
list(APPEND _protobuf_include_path -I ${ABS_PATH})
endif()
endforeach()
else()
set(_protobuf_include_path -I ${CMAKE_CURRENT_SOURCE_DIR})
endif()
if(DEFINED PROTOBUF3_IMPORT_DIRS)
foreach(DIR ${PROTOBUF3_IMPORT_DIRS})
get_filename_component(ABS_PATH ${DIR} ABSOLUTE)
list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)
if(${_contains_already} EQUAL -1)
list(APPEND _protobuf_include_path -I ${ABS_PATH})
endif()
endforeach()
endif()
set(${SRCS})
set(${HDRS})
foreach(FIL ${ARGN})
get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
get_filename_component(FIL_WE ${FIL} NAME_WE)
list(APPEND ${SRCS} "${DEST}/${FIL_WE}.grpc.pb.cc")
list(APPEND ${HDRS} "${DEST}/${FIL_WE}.grpc.pb.h")
add_custom_command(
OUTPUT "${DEST}/${FIL_WE}.grpc.pb.cc"
"${DEST}/${FIL_WE}.grpc.pb.h"
COMMAND ${PROTOBUF3_PROTOC3_EXECUTABLE}
ARGS --grpc_out ${DEST} ${_protobuf_include_path} --plugin=protoc-gen-grpc=${GRPC_CPP_PLUGIN} ${ABS_FIL}
DEPENDS ${ABS_FIL} ${PROTOBUF3_PROTOC3_EXECUTABLE} gRPC::grpc_cpp_plugin
COMMENT "Running C++ gRPC compiler on ${FIL}"
VERBATIM )
endforeach()
set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE)
set(${SRCS} ${${SRCS}} PARENT_SCOPE)
set(${HDRS} ${${HDRS}} PARENT_SCOPE)
endfunction()
# By default have GRPC_GENERATE_CPP macro pass -I to protoc
# for each directory where a proto file is referenced.
if(NOT DEFINED GRPC_GENERATE_CPP_APPEND_PATH)
set(GRPC_GENERATE_CPP_APPEND_PATH TRUE)
endif()
# Find gRPC include directory
find_path(GRPC_INCLUDE_DIR grpc/grpc.h)
mark_as_advanced(GRPC_INCLUDE_DIR)
# Find gRPC library
find_library(GRPC_LIBRARY NAMES grpc)
mark_as_advanced(GRPC_LIBRARY)
add_library(gRPC::grpc UNKNOWN IMPORTED)
set_target_properties(gRPC::grpc PROPERTIES
INTERFACE_INCLUDE_DIRECTORIES ${GRPC_INCLUDE_DIR}
INTERFACE_LINK_LIBRARIES "-lpthread;-ldl"
IMPORTED_LOCATION ${GRPC_LIBRARY}
)
# Find gRPC C++ library
find_library(GRPC_GRPC++_LIBRARY NAMES grpc++)
mark_as_advanced(GRPC_GRPC++_LIBRARY)
add_library(gRPC::grpc++ UNKNOWN IMPORTED)
set_target_properties(gRPC::grpc++ PROPERTIES
INTERFACE_INCLUDE_DIRECTORIES ${GRPC_INCLUDE_DIR}
INTERFACE_LINK_LIBRARIES gRPC::grpc
IMPORTED_LOCATION ${GRPC_GRPC++_LIBRARY}
)
# Find gRPC C++ reflection library
find_library(GRPC_GRPC++_REFLECTION_LIBRARY NAMES grpc++_reflection)
mark_as_advanced(GRPC_GRPC++_REFLECTION_LIBRARY)
add_library(gRPC::grpc++_reflection UNKNOWN IMPORTED)
set_target_properties(gRPC::grpc++_reflection PROPERTIES
INTERFACE_INCLUDE_DIRECTORIES ${GRPC_INCLUDE_DIR}
INTERFACE_LINK_LIBRARIES gRPC::grpc++
IMPORTED_LOCATION ${GRPC_GRPC++_REFLECTION_LIBRARY}
)
# Find gRPC CPP generator
find_program(GRPC_CPP_PLUGIN NAMES grpc_cpp_plugin)
mark_as_advanced(GRPC_CPP_PLUGIN)
add_executable(gRPC::grpc_cpp_plugin IMPORTED)
set_target_properties(gRPC::grpc_cpp_plugin PROPERTIES
IMPORTED_LOCATION ${GRPC_CPP_PLUGIN}
)
include(${CMAKE_ROOT}/Modules/FindPackageHandleStandardArgs.cmake)
FIND_PACKAGE_HANDLE_STANDARD_ARGS(gRPC DEFAULT_MSG
GRPC_LIBRARY GRPC_INCLUDE_DIR GRPC_GRPC++_REFLECTION_LIBRARY GRPC_CPP_PLUGIN)
# The CERN Tape Archive (CTA) project
# Copyright 2018 CERN
# Copyright 2019 CERN
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
......@@ -16,18 +16,33 @@
cmake_minimum_required(VERSION 2.6)
find_package(Protobuf3 REQUIRED)
set(PROTOBUF3_INCLUDE_PATH ${XRD_SSI_PB_DIR}/eos_cta/protobuf)
file(GLOB ProtoFiles "${PROTOBUF3_INCLUDE_PATH}/*.proto")
find_package(GRPC REQUIRED)
PROTOBUF3_GENERATE_CPP(ProtoSources ProtoHeaders ${ProtoFiles})
include_directories(${PROTOBUF3_INCLUDE_DIRS})
# Select protobuf files
set(PROTOBUF_EOS_CTA_DIR ${XRD_SSI_PB_DIR}/eos_cta/protobuf)
file(GLOB ProtoFilesEosCta "${PROTOBUF_EOS_CTA_DIR}/*.proto")
set(PROTOBUF_MIGRATION_DIR ${XRD_SSI_PB_DIR}/eos_cta/protobuf_grpc)
file(GLOB ProtoFilesMigration "${PROTOBUF_MIGRATION_DIR}/*.proto")
# Compile protobufs
set(PROTOBUF3_INCLUDE_PATH ${PROTOBUF_EOS_CTA_DIR}:${PROTOBUF_MIGRATION_DIR})
PROTOBUF3_GENERATE_CPP(ProtoSourcesEosCta ProtoHeadersEosCta ${ProtoFilesEosCta})
PROTOBUF3_GENERATE_CPP(ProtoSourcesMigration ProtoHeadersMigration ${ProtoFilesMigration})
# Compile gRPC code
set(GRPC_PROTOBUF_PATH "${CMAKE_BINARY_DIR}/eos_cta/")
grpc_generate_cpp(ProtoGrpcSourcesMigration ProtoGrpcHeadersMigration ${GRPC_PROTOBUF_PATH} ${ProtoFilesMigration})
set_source_files_properties(
${ProtoSources} ${ProtoHeaders}
PROPERTIES GENERATED 1)
${ProtoSourcesEosCta} ${ProtoHeadersEosCta}
${ProtoSourcesMigration} ${ProtoHeadersMigration}
${ProtoGrpcSourcesMigration} ${ProtoGrpcHeadersMigration}
PROPERTIES GENERATED TRUE)
foreach(PROTO_SRC ${ProtoSources})
foreach(PROTO_SRC ${ProtoSourcesEosCta} ${ProtoSourcesMigration} ${ProtoGrpcSourcesMigration})
set_property(SOURCE ${PROTO_SRC} PROPERTY COMPILE_FLAGS " -Wno-missing-field-initializers -fPIC -Wno-narrowing -Wno-implicit-fallthrough")
# Add -Wno-narrowing -Wno-implicit-fallthrough compiler flags if using gcc version 7 or greater
if(CMAKE_COMPILER_IS_GNUCC)
if(GCC_VERSION VERSION_EQUAL 7 OR GCC_VERSION VERSION_GREATER 7)
......@@ -35,9 +50,15 @@ foreach(PROTO_SRC ${ProtoSources})
endif(GCC_VERSION VERSION_EQUAL 7 OR GCC_VERSION VERSION_GREATER 7)
endif(CMAKE_COMPILER_IS_GNUCC)
endforeach(PROTO_SRC)
set(CTA_FRONT_END_MESSAGES_SRC_FILES ${ProtoSources})
include_directories(${PROTOBUF3_INCLUDE_DIRS})
add_library(XrdSsiPbEosCta ${CTA_FRONT_END_MESSAGES_SRC_FILES})
set_target_properties(XrdSsiPbEosCta PROPERTIES LINKER_LANGUAGE CXX)
add_library(XrdSsiPbEosCta ${ProtoSourcesEosCta})
set_target_properties(XrdSsiPbEosCta PROPERTIES
LINKER_LANGUAGE CXX
POSITION_INDEPENDENT_CODE TRUE)
target_link_libraries(XrdSsiPbEosCta ${PROTOBUF3_LIBRARIES})
add_library(EosMigration ${ProtoSourcesMigration} ${ProtoGrpcSourcesMigration})
set_target_properties(EosMigration PROPERTIES
LINKER_LANGUAGE CXX
POSITION_INDEPENDENT_CODE TRUE)
target_link_libraries(EosMigration ${PROTOBUF3_GRPC_LIBRARIES})
......@@ -16,9 +16,6 @@
cmake_minimum_required (VERSION 2.6)
add_subdirectory(gephi)
#add_subdirectory(not_on_tape)
include_directories(${XRD_SSI_PB_DIR}/include ${XROOTD_INCLUDE_DIR} ${XROOTD_INCLUDE_DIR}/private
${CMAKE_BINARY_DIR}/eos_cta
${PROTOBUF3_INCLUDE_DIRS})
......@@ -28,5 +25,5 @@ add_executable(eos-import-dirs EosImportDirs.cpp GrpcClient.cpp)
target_link_libraries(eos-import-dirs EosMigration ${PROTOBUF3_LIBRARIES} ${GRPC_LIBRARY} ${GRPC_GRPC++_LIBRARY} ctacatalogue)
set_property(TARGET eos-import-dirs APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH})
#install(TARGETS eos-import-dirs DESTINATION usr/bin)
#install(FILES eos-import-dirs.1cta DESTINATION usr/share/man/man1)
install(TARGETS eos-import-dirs DESTINATION usr/bin)
install(FILES eos-import-dirs.1cta DESTINATION usr/share/man/man1)
......@@ -20,7 +20,7 @@
#include <sstream>
#include <XrdSsiPbConfig.hpp>
#include "CastorDB.hpp"
#include <upgrade_db/OracleDbConn.hpp>
#include "GrpcClient.hpp"
......@@ -29,8 +29,7 @@ namespace cta {
namespace migration {
//! DB Connection Pool
std::unique_ptr<rdbms::ConnPool> CastorDbConn::m_connPool;
std::unique_ptr<rdbms::ConnPool> OracleDbConn::m_connPool;
class EosImportDirs
......@@ -46,28 +45,23 @@ private:
std::chrono::steady_clock::time_point m_start_time; //!< Start the clock
unsigned int m_total_dirs; //!< Count how many have been processed
CastorDbConn m_castordb; //!< Oracle database for CASTOR namespace
unsigned int m_max_depth; //!< Maximum directory tree depth to import
unsigned int m_cur_depth; //!< Current directory tree depth
unsigned int m_batch_size; //!< Number of records to fetch from the DB at a time
OracleDbConn m_castordb; //!< Oracle database for CASTOR->CTA migration tables
unsigned int m_batch_size; //!< Number of records to fetch from the DB at a time
std::unique_ptr<eos::client::GrpcClient> m_eosgrpc; //!< EOS gRPC API interface
std::string m_prefix; //!< EOS namespace prefix
};
EosImportDirs::EosImportDirs(const std::string &configfile) :
m_start_time(std::chrono::steady_clock::now()),
m_total_dirs(0),
m_cur_depth(0)
m_total_dirs(0)
{
// Parse configuration file
XrdSsiPb::Config config(configfile);
auto dbconn = config.getOptionValueStr("castor.db_login");
auto max_num_conns = config.getOptionValueInt("castor.max_num_connections");
auto max_depth = config.getOptionValueInt("castor.max_depth");
auto batch_size = config.getOptionValueInt("castor.batch_size");
auto endpoint = config.getOptionValueStr("eos.endpoint");
auto token = config.getOptionValueStr("eos.token");
......@@ -85,49 +79,48 @@ EosImportDirs::EosImportDirs(const std::string &configfile) :
token.second);
// Set parameters and defaults
m_max_depth = max_depth.first ? max_depth.second : std::numeric_limits<unsigned int>::max();;
m_batch_size = batch_size.first ? batch_size.second : 1000;
m_prefix = prefix.first ? prefix.second : "/eos";
m_batch_size = batch_size.first ? batch_size.second : 10000;
m_prefix = prefix.first ? prefix.second : "/eos/grpc";
}
bool EosImportDirs::getNextBatch(std::vector<std::string> &paths)
{
for(unsigned int b = 0; b < m_batch_size; ++b) {
if(m_castordb.isQueryEmpty() || !m_castordb.nextRow()) {
if(m_cur_depth > m_max_depth) return false;
m_castordb.query("select name from mdavis_cns_dirnames where depth=" + std::to_string(m_cur_depth++));
m_castordb.query("select fileid, path, disk_uid, disk_gid, filemode, atime, mtime, classid from ctadirshelper");
if(m_castordb.isQueryEmpty()) return false;
}
paths.push_back(m_prefix + m_castordb.getResultColumnString("NAME"));
paths.push_back(m_prefix + m_castordb.getResultColumnString("PATH"));
++m_total_dirs;
}
return true;
}
bool EosImportDirs::processBatch()
{
std::vector<std::string> paths;
bool isFinished = getNextBatch(paths);
if(!paths.empty()) {
std::cout << paths.front() << std::endl;
#if 0
int retc = m_eosgrpc->ContainerInsert(paths);
if(retc != 0) {
throw std::runtime_error("EosImportDirs::processBatch(): ContainerInsert failed with error " +
std::to_string(retc));
}
#endif
auto elapsed_time = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::steady_clock::now() - m_start_time);
std::cout << "Processed " << m_total_dirs << " directories in " << elapsed_time.count() << "s" << std::endl;
}
isFinished=true;
return isFinished;
}
}} // namespace cta::migration
void throwUsage(const std::string &program, const std::string &error_txt)
{
std::stringstream help;
......@@ -139,10 +132,9 @@ void throwUsage(const std::string &program, const std::string &error_txt)
}
int main(int argc, const char* argv[])
{
std::string configfile = "/home/ctadev/CTA/migration/castor-migration.conf";
std::string configfile = "/etc/cta/castor-migration.conf";
try {
for(auto i = 1; i < argc; ++i) {
......
......@@ -292,7 +292,7 @@ void throwUsage(const std::string &program, const std::string &error_txt)
int main(int argc, const char* argv[])
{
std::string configfile = "/home/ctadev/CTA/migration/cta-catalogue-migration.conf";
std::string configfile = "/etc/cta/cta-catalogue-upgrade.conf";
bool renameLogical = false;
bool undoRenameLogical = false;
......
#
# Configuration for migration tools
# Configuration for CTA database update tool
#
cta.db_login oracle:<USER>/<PASSWORD>@<DATABASE>
cta.max_num_connections 1
cta.batch_size 10000
Subproject commit 703aa091b704c14e2a983f92b685e1d82e063bfa
Subproject commit 5e5cb0b302c4225d8e47c2b788a1d91ce4fa57f5
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment