[migration] Separates DB migration tools from DB upgrade tools

- CASTOR migration tools stay under CTA/migration
- upgrade tools are now under CTA/upgrade_db
- adds FindGRPC.cmake (required for migration tools)
- updates submodule to v0.50 (required for EOS gRPC API)

CMakeLists.txt

@@ -148,7 +148,8 @@ ELSE(DEFINED PackageOnly)
   add_subdirectory(tapeserver)
   add_subdirectory(XRootdSSiRmcd)
 
-  add_subdirectory(migration/CTA)
+  add_subdirectory(upgrade_db)
+  add_subdirectory(migration/gRPC)
 
   #Generate version information
   configure_file(${PROJECT_SOURCE_DIR}/version.hpp.in

cmake/FindGRPC.cmake

+#
+# Locate and configure the gRPC library
+#
+# Adds the following targets:
+#
+#  gRPC::grpc - gRPC library
+#  gRPC::grpc++ - gRPC C++ library
+#  gRPC::grpc++_reflection - gRPC C++ reflection library
+#  gRPC::grpc_cpp_plugin - C++ generator plugin for Protocol Buffers
+#
+
+#
+# Generates C++ sources from the .proto files
+#
+# grpc_generate_cpp (<SRCS> <HDRS> <DEST> [<ARGN>...])
+#
+#  SRCS - variable to define with autogenerated source files
+#  HDRS - variable to define with autogenerated header files
+#  DEST - directory where the source files will be created
+#  ARGN - .proto files
+#
+function(GRPC_GENERATE_CPP SRCS HDRS DEST)
+  if(NOT ARGN)
+    message(SEND_ERROR "Error: GRPC_GENERATE_CPP() called without any proto files")
+    return()
+  endif()
+
+  if(GRPC_GENERATE_CPP_APPEND_PATH)
+    # Create an include path for each file specified
+    foreach(FIL ${ARGN})
+      get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+      get_filename_component(ABS_PATH ${ABS_FIL} PATH)
+      list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)
+      if(${_contains_already} EQUAL -1)
+          list(APPEND _protobuf_include_path -I ${ABS_PATH})
+      endif()
+    endforeach()
+  else()
+    set(_protobuf_include_path -I ${CMAKE_CURRENT_SOURCE_DIR})
+  endif()
+
+  if(DEFINED PROTOBUF3_IMPORT_DIRS)
+    foreach(DIR ${PROTOBUF3_IMPORT_DIRS})
+      get_filename_component(ABS_PATH ${DIR} ABSOLUTE)
+      list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)
+      if(${_contains_already} EQUAL -1)
+          list(APPEND _protobuf_include_path -I ${ABS_PATH})
+      endif()
+    endforeach()
+  endif()
+
+  set(${SRCS})
+  set(${HDRS})
+  foreach(FIL ${ARGN})
+    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+    get_filename_component(FIL_WE ${FIL} NAME_WE)
+
+    list(APPEND ${SRCS} "${DEST}/${FIL_WE}.grpc.pb.cc")
+    list(APPEND ${HDRS} "${DEST}/${FIL_WE}.grpc.pb.h")
+
+    add_custom_command(
+      OUTPUT "${DEST}/${FIL_WE}.grpc.pb.cc"
+             "${DEST}/${FIL_WE}.grpc.pb.h"
+      COMMAND ${PROTOBUF3_PROTOC3_EXECUTABLE}
+      ARGS --grpc_out ${DEST} ${_protobuf_include_path} --plugin=protoc-gen-grpc=${GRPC_CPP_PLUGIN} ${ABS_FIL}
+      DEPENDS ${ABS_FIL} ${PROTOBUF3_PROTOC3_EXECUTABLE} gRPC::grpc_cpp_plugin
+      COMMENT "Running C++ gRPC compiler on ${FIL}"
+      VERBATIM )
+  endforeach()
+
+  set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE)
+  set(${SRCS} ${${SRCS}} PARENT_SCOPE)
+  set(${HDRS} ${${HDRS}} PARENT_SCOPE)
+endfunction()
+
+# By default have GRPC_GENERATE_CPP macro pass -I to protoc
+# for each directory where a proto file is referenced.
+if(NOT DEFINED GRPC_GENERATE_CPP_APPEND_PATH)
+  set(GRPC_GENERATE_CPP_APPEND_PATH TRUE)
+endif()
+
+# Find gRPC include directory
+find_path(GRPC_INCLUDE_DIR grpc/grpc.h)
+mark_as_advanced(GRPC_INCLUDE_DIR)
+
+# Find gRPC library
+find_library(GRPC_LIBRARY NAMES grpc)
+mark_as_advanced(GRPC_LIBRARY)
+add_library(gRPC::grpc UNKNOWN IMPORTED)
+set_target_properties(gRPC::grpc PROPERTIES
+    INTERFACE_INCLUDE_DIRECTORIES ${GRPC_INCLUDE_DIR}
+    INTERFACE_LINK_LIBRARIES "-lpthread;-ldl"
+    IMPORTED_LOCATION ${GRPC_LIBRARY}
+)
+
+# Find gRPC C++ library
+find_library(GRPC_GRPC++_LIBRARY NAMES grpc++)
+mark_as_advanced(GRPC_GRPC++_LIBRARY)
+add_library(gRPC::grpc++ UNKNOWN IMPORTED)
+set_target_properties(gRPC::grpc++ PROPERTIES
+    INTERFACE_INCLUDE_DIRECTORIES ${GRPC_INCLUDE_DIR}
+    INTERFACE_LINK_LIBRARIES gRPC::grpc
+    IMPORTED_LOCATION ${GRPC_GRPC++_LIBRARY}
+)
+
+# Find gRPC C++ reflection library
+find_library(GRPC_GRPC++_REFLECTION_LIBRARY NAMES grpc++_reflection)
+mark_as_advanced(GRPC_GRPC++_REFLECTION_LIBRARY)
+add_library(gRPC::grpc++_reflection UNKNOWN IMPORTED)
+set_target_properties(gRPC::grpc++_reflection PROPERTIES
+    INTERFACE_INCLUDE_DIRECTORIES ${GRPC_INCLUDE_DIR}
+    INTERFACE_LINK_LIBRARIES gRPC::grpc++
+    IMPORTED_LOCATION ${GRPC_GRPC++_REFLECTION_LIBRARY}
+)
+
+# Find gRPC CPP generator
+find_program(GRPC_CPP_PLUGIN NAMES grpc_cpp_plugin)
+mark_as_advanced(GRPC_CPP_PLUGIN)
+add_executable(gRPC::grpc_cpp_plugin IMPORTED)
+set_target_properties(gRPC::grpc_cpp_plugin PROPERTIES
+    IMPORTED_LOCATION ${GRPC_CPP_PLUGIN}
+)
+
+include(${CMAKE_ROOT}/Modules/FindPackageHandleStandardArgs.cmake)
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(gRPC DEFAULT_MSG
+    GRPC_LIBRARY GRPC_INCLUDE_DIR GRPC_GRPC++_REFLECTION_LIBRARY GRPC_CPP_PLUGIN)

eos_cta/CMakeLists.txt

 # The CERN Tape Archive (CTA) project
-# Copyright 2018 CERN
+# Copyright 2019 CERN
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -16,18 +16,33 @@
 cmake_minimum_required(VERSION 2.6)
 
 find_package(Protobuf3 REQUIRED)
-set(PROTOBUF3_INCLUDE_PATH ${XRD_SSI_PB_DIR}/eos_cta/protobuf)
-file(GLOB ProtoFiles "${PROTOBUF3_INCLUDE_PATH}/*.proto")
+find_package(GRPC REQUIRED)
 
-PROTOBUF3_GENERATE_CPP(ProtoSources ProtoHeaders ${ProtoFiles})
+include_directories(${PROTOBUF3_INCLUDE_DIRS})
+
+# Select protobuf files
+set(PROTOBUF_EOS_CTA_DIR ${XRD_SSI_PB_DIR}/eos_cta/protobuf)
+file(GLOB ProtoFilesEosCta "${PROTOBUF_EOS_CTA_DIR}/*.proto")
+set(PROTOBUF_MIGRATION_DIR ${XRD_SSI_PB_DIR}/eos_cta/protobuf_grpc)
+file(GLOB ProtoFilesMigration "${PROTOBUF_MIGRATION_DIR}/*.proto")
+
+# Compile protobufs
+set(PROTOBUF3_INCLUDE_PATH ${PROTOBUF_EOS_CTA_DIR}:${PROTOBUF_MIGRATION_DIR})
+PROTOBUF3_GENERATE_CPP(ProtoSourcesEosCta ProtoHeadersEosCta ${ProtoFilesEosCta})
+PROTOBUF3_GENERATE_CPP(ProtoSourcesMigration ProtoHeadersMigration ${ProtoFilesMigration})
+
+# Compile gRPC code
+set(GRPC_PROTOBUF_PATH "${CMAKE_BINARY_DIR}/eos_cta/")
+grpc_generate_cpp(ProtoGrpcSourcesMigration ProtoGrpcHeadersMigration ${GRPC_PROTOBUF_PATH} ${ProtoFilesMigration})
 
 set_source_files_properties(
-  ${ProtoSources} ${ProtoHeaders}
-  PROPERTIES GENERATED 1)
+  ${ProtoSourcesEosCta} ${ProtoHeadersEosCta}
+  ${ProtoSourcesMigration} ${ProtoHeadersMigration}
+  ${ProtoGrpcSourcesMigration} ${ProtoGrpcHeadersMigration}
+  PROPERTIES GENERATED TRUE)
 
-foreach(PROTO_SRC ${ProtoSources})
+foreach(PROTO_SRC ${ProtoSourcesEosCta} ${ProtoSourcesMigration} ${ProtoGrpcSourcesMigration})
   set_property(SOURCE ${PROTO_SRC} PROPERTY COMPILE_FLAGS " -Wno-missing-field-initializers -fPIC -Wno-narrowing -Wno-implicit-fallthrough")
-
   # Add -Wno-narrowing -Wno-implicit-fallthrough compiler flags if using gcc version 7 or greater
   if(CMAKE_COMPILER_IS_GNUCC)
     if(GCC_VERSION VERSION_EQUAL 7 OR GCC_VERSION VERSION_GREATER 7)
@@ -35,9 +50,15 @@ foreach(PROTO_SRC ${ProtoSources})
     endif(GCC_VERSION VERSION_EQUAL 7 OR GCC_VERSION VERSION_GREATER 7)
   endif(CMAKE_COMPILER_IS_GNUCC)
 endforeach(PROTO_SRC)
-set(CTA_FRONT_END_MESSAGES_SRC_FILES ${ProtoSources})
 
-include_directories(${PROTOBUF3_INCLUDE_DIRS})
-add_library(XrdSsiPbEosCta ${CTA_FRONT_END_MESSAGES_SRC_FILES})
-set_target_properties(XrdSsiPbEosCta PROPERTIES LINKER_LANGUAGE CXX)
+add_library(XrdSsiPbEosCta ${ProtoSourcesEosCta})
+set_target_properties(XrdSsiPbEosCta PROPERTIES
+  LINKER_LANGUAGE CXX
+  POSITION_INDEPENDENT_CODE TRUE)
 target_link_libraries(XrdSsiPbEosCta ${PROTOBUF3_LIBRARIES})
+
+add_library(EosMigration ${ProtoSourcesMigration} ${ProtoGrpcSourcesMigration})
+set_target_properties(EosMigration PROPERTIES
+  LINKER_LANGUAGE CXX
+  POSITION_INDEPENDENT_CODE TRUE)
+target_link_libraries(EosMigration ${PROTOBUF3_GRPC_LIBRARIES})

migration/gRPC/CMakeLists.txt

@@ -16,9 +16,6 @@
 
 cmake_minimum_required (VERSION 2.6)
 
-add_subdirectory(gephi)
-#add_subdirectory(not_on_tape)
-
 include_directories(${XRD_SSI_PB_DIR}/include ${XROOTD_INCLUDE_DIR} ${XROOTD_INCLUDE_DIR}/private
                     ${CMAKE_BINARY_DIR}/eos_cta
                     ${PROTOBUF3_INCLUDE_DIRS})
@@ -28,5 +25,5 @@ add_executable(eos-import-dirs EosImportDirs.cpp GrpcClient.cpp)
 target_link_libraries(eos-import-dirs EosMigration ${PROTOBUF3_LIBRARIES} ${GRPC_LIBRARY} ${GRPC_GRPC++_LIBRARY} ctacatalogue)
 set_property(TARGET eos-import-dirs APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH})
 
-#install(TARGETS eos-import-dirs DESTINATION usr/bin)
-#install(FILES eos-import-dirs.1cta DESTINATION usr/share/man/man1)
+install(TARGETS eos-import-dirs DESTINATION usr/bin)
+install(FILES eos-import-dirs.1cta DESTINATION usr/share/man/man1)

migration/gRPC/EosImportDirs.cpp

@@ -20,7 +20,7 @@
 #include <sstream>
 
 #include <XrdSsiPbConfig.hpp>
-#include "CastorDB.hpp"
+#include <upgrade_db/OracleDbConn.hpp>
 #include "GrpcClient.hpp"
 
 
@@ -29,8 +29,7 @@ namespace cta {
 namespace migration {
 
 //! DB Connection Pool
-std::unique_ptr<rdbms::ConnPool> CastorDbConn::m_connPool;
-
+std::unique_ptr<rdbms::ConnPool> OracleDbConn::m_connPool;
 
 
 class EosImportDirs
@@ -46,28 +45,23 @@ private:
   std::chrono::steady_clock::time_point m_start_time;    //!< Start the clock
   unsigned int m_total_dirs;                             //!< Count how many have been processed
 
-  CastorDbConn m_castordb;      //!< Oracle database for CASTOR namespace
-  unsigned int m_max_depth;     //!< Maximum directory tree depth to import
-  unsigned int m_cur_depth;     //!< Current directory tree depth
-  unsigned int m_batch_size;    //!< Number of records to fetch from the DB at a time
+  OracleDbConn m_castordb;                               //!< Oracle database for CASTOR->CTA migration tables
+  unsigned int m_batch_size;                             //!< Number of records to fetch from the DB at a time
 
   std::unique_ptr<eos::client::GrpcClient> m_eosgrpc;    //!< EOS gRPC API interface
   std::string m_prefix;                                  //!< EOS namespace prefix
 };
 
 
-
 EosImportDirs::EosImportDirs(const std::string &configfile) :
   m_start_time(std::chrono::steady_clock::now()),
-  m_total_dirs(0),
-  m_cur_depth(0)
+  m_total_dirs(0)
 {
   // Parse configuration file
   XrdSsiPb::Config config(configfile);
 
   auto dbconn        = config.getOptionValueStr("castor.db_login");
   auto max_num_conns = config.getOptionValueInt("castor.max_num_connections");
-  auto max_depth     = config.getOptionValueInt("castor.max_depth");
   auto batch_size    = config.getOptionValueInt("castor.batch_size");
   auto endpoint      = config.getOptionValueStr("eos.endpoint");
   auto token         = config.getOptionValueStr("eos.token");
@@ -85,49 +79,48 @@ EosImportDirs::EosImportDirs(const std::string &configfile) :
     token.second);
 
   // Set parameters and defaults
-  m_max_depth  = max_depth.first  ? max_depth.second  : std::numeric_limits<unsigned int>::max();;
-  m_batch_size = batch_size.first ? batch_size.second : 1000;
-  m_prefix     = prefix.first     ? prefix.second     : "/eos";
+  m_batch_size = batch_size.first ? batch_size.second : 10000;
+  m_prefix     = prefix.first     ? prefix.second     : "/eos/grpc";
 }
 
 
-
 bool EosImportDirs::getNextBatch(std::vector<std::string> &paths)
 {
   for(unsigned int b = 0; b < m_batch_size; ++b) {
     if(m_castordb.isQueryEmpty() || !m_castordb.nextRow()) {
-      if(m_cur_depth > m_max_depth) return false;
-      m_castordb.query("select name from mdavis_cns_dirnames where depth=" + std::to_string(m_cur_depth++));
+      m_castordb.query("select fileid, path, disk_uid, disk_gid, filemode, atime, mtime, classid from ctadirshelper");
       if(m_castordb.isQueryEmpty()) return false;
     }
-    paths.push_back(m_prefix + m_castordb.getResultColumnString("NAME"));
+    paths.push_back(m_prefix + m_castordb.getResultColumnString("PATH"));
     ++m_total_dirs;
   }
   return true;
 }
 
 
-
 bool EosImportDirs::processBatch()
 {
   std::vector<std::string> paths;
   bool isFinished = getNextBatch(paths);
   if(!paths.empty()) {
+std::cout << paths.front() << std::endl;
+#if 0
     int retc = m_eosgrpc->ContainerInsert(paths);
     if(retc != 0) {
       throw std::runtime_error("EosImportDirs::processBatch(): ContainerInsert failed with error " +
         std::to_string(retc));
     }
+#endif
     auto elapsed_time = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::steady_clock::now() - m_start_time);
     std::cout << "Processed " << m_total_dirs << " directories in " << elapsed_time.count() << "s" << std::endl;
   }
+isFinished=true;
   return isFinished;
 }
 
 }} // namespace cta::migration
 
 
-
 void throwUsage(const std::string &program, const std::string &error_txt)
 {
   std::stringstream help;
@@ -139,10 +132,9 @@ void throwUsage(const std::string &program, const std::string &error_txt)
 }
 
 
-
 int main(int argc, const char* argv[])
 {
-  std::string configfile = "/home/ctadev/CTA/migration/castor-migration.conf";
+  std::string configfile = "/etc/cta/castor-migration.conf";
 
   try {
     for(auto i = 1; i < argc; ++i) {

migration/CTA/UpgradeDB.cpp → upgrade_db/UpgradeDB.cpp

@@ -292,7 +292,7 @@ void throwUsage(const std::string &program, const std::string &error_txt)
 
 int main(int argc, const char* argv[])
 {
-  std::string configfile = "/home/ctadev/CTA/migration/cta-catalogue-migration.conf";
+  std::string configfile = "/etc/cta/cta-catalogue-upgrade.conf";
 
   bool renameLogical = false;
   bool undoRenameLogical = false;

migration/CTA/cta-catalogue-migration.conf → upgrade_db/cta-catalogue-upgrade.conf

 #
-# Configuration for migration tools
+# Configuration for CTA database update tool
 #
-
 cta.db_login               oracle:<USER>/<PASSWORD>@<DATABASE>
-cta.max_num_connections    1
-cta.batch_size             10000

xrootd-ssi-protobuf-interface @ 5e5cb0b3

-Subproject commit 703aa091b704c14e2a983f92b685e1d82e063bfa
+Subproject commit 5e5cb0b302c4225d8e47c2b788a1d91ce4fa57f5