[migration] Initial version of UpgradeDB tool

CMakeLists.txt

@@ -147,7 +147,9 @@ ELSE(DEFINED PackageOnly)
   add_subdirectory(scheduler)
   add_subdirectory(tapeserver)
   add_subdirectory(XRootdSSiRmcd)
-  
+
+  add_subdirectory(migration/CTA)
+
   #Generate version information
   configure_file(${PROJECT_SOURCE_DIR}/version.hpp.in
     ${CMAKE_BINARY_DIR}/version.h)

migration/CTA/CMakeLists.txt

+# The CERN Tape Archive (CTA) project
+# Copyright 2019 CERN
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+cmake_minimum_required (VERSION 2.6)
+
+include_directories(${XRD_SSI_PB_DIR}/include ${XROOTD_INCLUDE_DIR} ${XROOTD_INCLUDE_DIR}/private)
+
+# Upgrade DB tool
+add_executable(upgrade-db UpgradeDB.cpp)
+target_link_libraries(upgrade-db ctacatalogue)
+
+#install(TARGETS upgrade-db DESTINATION usr/bin)

migration/CTA/OracleDbConn.hpp

+/*!
+ * @project        The CERN Tape Archive (CTA)
+ * @brief          Access Oracle DB for migration operations
+ * @copyright      Copyright 2019 CERN
+ * @license        This program is free software: you can redistribute it and/or modify
+ *                 it under the terms of the GNU General Public License as published by
+ *                 the Free Software Foundation, either version 3 of the License, or
+ *                 (at your option) any later version.
+ *
+ *                 This program is distributed in the hope that it will be useful,
+ *                 but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *                 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *                 GNU General Public License for more details.
+ *
+ *                 You should have received a copy of the GNU General Public License
+ *                 along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <memory>
+#include <rdbms/Login.hpp>
+#include <rdbms/ConnPool.hpp>
+
+
+
+namespace cta {
+namespace migration {
+
+/*!
+ * Manage a single database query over a single database connection
+ */
+class OracleDbConn
+{
+public:
+  OracleDbConn() : m_queryIsEmpty(true) {}
+
+  void connect(const std::string &dbconn = "", unsigned int max_num_conns = 1) {
+    // Initialise the connection pool
+    if(!dbconn.empty()) {
+      m_connPool.reset(new rdbms::ConnPool(rdbms::Login::parseString(dbconn), max_num_conns));
+    }
+
+    // Initialise this connection
+    m_conn = m_connPool->getConn();
+  }
+
+  void query(const std::string &sqlString) {
+    auto sql = m_conn.createStmt(sqlString);
+    m_rSet = sql.executeQuery();
+    m_queryIsEmpty = !m_rSet.next();
+  }
+
+  bool nextRow() {
+    return m_rSet.next();
+  }
+
+  std::string getResultColumnString(const std::string &col) const {
+    return m_rSet.columnString(col);
+  }
+
+  bool isQueryEmpty() const {
+    return m_queryIsEmpty;
+  }
+
+private:
+  static std::unique_ptr<rdbms::ConnPool> m_connPool;    //!< The pool of connections to the database
+  rdbms::Conn m_conn;                                    //!< The connection we are using
+  rdbms::Rset m_rSet;                                    //!< Result set for the last query executed
+  bool m_queryIsEmpty;                                   //!< Track whether the last query had an empty result set
+};
+
+}} // namespace cta::migration

migration/CTA/UpgradeDB.cpp

+/*!
+ * @project        The CERN Tape Archive (CTA)
+ * @brief          Migration tool to upgrade Oracle DB
+ * @copyright      Copyright 2019 CERN
+ * @license        This program is free software: you can redistribute it and/or modify
+ *                 it under the terms of the GNU General Public License as published by
+ *                 the Free Software Foundation, either version 3 of the License, or
+ *                 (at your option) any later version.
+ *
+ *                 This program is distributed in the hope that it will be useful,
+ *                 but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *                 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *                 GNU General Public License for more details.
+ *
+ *                 You should have received a copy of the GNU General Public License
+ *                 along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <iostream>
+#include <sstream>
+#include <vector>
+
+#include <XrdSsiPbConfig.hpp>
+#include "OracleDbConn.hpp"
+
+namespace cta {
+namespace migration {
+
+//! DB Connection Pool
+std::unique_ptr<rdbms::ConnPool> OracleDbConn::m_connPool;
+
+
+
+class UpgradeDB
+{
+public:
+  UpgradeDB(const std::string &configfile);
+
+  bool processBatch();
+
+private:
+  bool getNextBatch(std::vector<std::string> &paths);
+
+  OracleDbConn m_ctadb;         //!< Oracle database for CTA Catalogue
+  unsigned int m_max_depth;     //!< Maximum directory tree depth to import
+  unsigned int m_cur_depth;     //!< Current directory tree depth
+  unsigned int m_batch_size;    //!< Number of records to fetch from the DB at a time
+};
+
+
+
+UpgradeDB::UpgradeDB(const std::string &configfile)
+{
+  // Parse configuration file
+  XrdSsiPb::Config config(configfile);
+
+  auto dbconn        = config.getOptionValueStr("cta.db_login");
+  auto max_num_conns = config.getOptionValueInt("cta.max_num_connections");
+  auto max_depth     = config.getOptionValueInt("cta.max_depth");
+  auto batch_size    = config.getOptionValueInt("cta.batch_size");
+
+  // Connect to Oracle
+  if(!dbconn.first) {
+    throw std::runtime_error("castor.db_login must be specified in the config file in the form oracle:user/password@TNS");
+  }
+  m_ctadb.connect(dbconn.second, max_num_conns.first ? max_num_conns.second : 1);
+
+  // Set parameters and defaults
+  m_max_depth  = max_depth.first  ? max_depth.second  : std::numeric_limits<unsigned int>::max();;
+  m_batch_size = batch_size.first ? batch_size.second : 1000;
+}
+
+
+
+bool UpgradeDB::getNextBatch(std::vector<std::string> &paths)
+{
+#if 0
+  for(unsigned int b = 0; b < m_batch_size; ++b) {
+    if(m_ctadb.isQueryEmpty() || !m_ctadb.nextRow()) {
+      if(m_cur_depth > m_max_depth) return false;
+      m_ctadb.query("select name from mdavis_cns_dirnames where depth=" + std::to_string(m_cur_depth++));
+      if(m_ctadb.isQueryEmpty()) return false;
+    }
+    paths.push_back(m_prefix + m_ctadb.getResultColumnString("NAME"));
+    ++m_total_dirs;
+  }
+#endif
+  return true;
+}
+
+
+
+bool UpgradeDB::processBatch()
+{
+#if 0
+  std::vector<std::string> paths;
+  bool isFinished = getNextBatch(paths);
+  if(!paths.empty()) {
+    int retc = m_eosgrpc->ContainerInsert(paths);
+    if(retc != 0) {
+      throw std::runtime_error("UpgradeDB::processBatch(): ContainerInsert failed with error " +
+        std::to_string(retc));
+    }
+    auto elapsed_time = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::steady_clock::now() - m_start_time);
+    std::cout << "Processed " << m_total_dirs << " directories in " << elapsed_time.count() << "s" << std::endl;
+  }
+  return isFinished;
+#endif
+  return true;
+}
+
+}} // namespace cta::migration
+
+
+
+void throwUsage(const std::string &program, const std::string &error_txt)
+{
+  std::stringstream help;
+
+  help << program << ": " << error_txt << std::endl
+       << "Usage: " << program << " [--config <config_file>]";
+
+  throw std::runtime_error(help.str());
+}
+
+
+
+int main(int argc, const char* argv[])
+{
+  std::string configfile = "/home/ctadev/CTA/migration/castor-migration.conf";
+
+  try {
+    for(auto i = 1; i < argc; ++i) {
+      std::string option(argv[i]);
+
+      if(option == "--config" && argc > ++i) {
+        configfile = argv[i];
+        continue;
+      }
+      throwUsage(argv[0], "invalid option " + option);
+    }
+    cta::migration::UpgradeDB importDirs(configfile);
+    while(importDirs.processBatch()) ;
+  } catch(std::runtime_error &ex) {
+    std::cerr << ex.what() << std::endl;
+    return -1;
+  }
+  return 0;
+}
+

migration/gRPC/CMakeLists.txt

+# The CERN Tape Archive (CTA) project
+# Copyright 2019 CERN
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+cmake_minimum_required (VERSION 2.6)
+
+add_subdirectory(gephi)
+#add_subdirectory(not_on_tape)
+
+include_directories(${XRD_SSI_PB_DIR}/include ${XROOTD_INCLUDE_DIR} ${XROOTD_INCLUDE_DIR}/private
+                    ${CMAKE_BINARY_DIR}/eos_cta
+                    ${PROTOBUF3_INCLUDE_DIRS})
+
+# Directory import tool
+add_executable(eos-import-dirs EosImportDirs.cpp GrpcClient.cpp)
+target_link_libraries(eos-import-dirs EosMigration ${PROTOBUF3_LIBRARIES} ${GRPC_LIBRARY} ${GRPC_GRPC++_LIBRARY} ctacatalogue)
+set_property(TARGET eos-import-dirs APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH})
+
+#install(TARGETS eos-import-dirs DESTINATION usr/bin)
+#install(FILES eos-import-dirs.1cta DESTINATION usr/share/man/man1)