Skip to content

GitLab

Commit 114b50c8 authored by Michael Davis's avatar Michael Davis
Browse files

[migration] Deletes eos-insert-missing-dirs (deprecated)

parent bf0b3187
Hide whitespace changes
Inline Side-by-side
migration/gRPC/CMakeLists.txt
View file @ 114b50c8
......@@ -34,12 +34,6 @@ target_link_libraries(eos-import-files EosMigration ${PROTOBUF3_LIBRARIES} ${GRP
set_property(TARGET eos-import-files APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH})
set_property(TARGET eos-import-files APPEND PROPERTY INSTALL_RPATH ${ORACLE-INSTANTCLIENT_RPATH})
## Tool to insert missing directories
#add_executable(eos-insert-missing-dirs EosInsertMissingDirs.cpp)
#target_link_libraries(eos-insert-missing-dirs ctacatalogue)
#set_property(TARGET eos-insert-missing-dirs APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH})
#set_property(TARGET eos-insert-missing-dirs APPEND PROPERTY INSTALL_RPATH ${ORACLE-INSTANTCLIENT_RPATH})
# Standalone test tools
#add_executable(grpc-utils-test GrpcUtilsTest.cpp GrpcUtils.cpp)
......
migration/gRPC/EosInsertMissingDirs.cpp deleted 100644 → 0
View file @ bf0b3187
/*!
* @project The CERN Tape Archive (CTA)
* @brief Inject directory metadata into EOS using gRPC
* @copyright Copyright 2019 CERN
* @license This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <iostream>
#include <sstream>
#include <XrdSsiPbConfig.hpp>
#include <XrdSsiPbLog.hpp>
#include "OracleDbConn.hpp"
namespace cta {
namespace migration {
//! DB Connection Pool
std::unique_ptr<rdbms::ConnPool> OracleDbConn::m_connPool;
class EosInsertMissingDirs
{
public:
EosInsertMissingDirs(const std::string &configfile);
void process();
struct Dirname {
std::string basename; //!< Just the directory name
std::string pathname; //!< The full path including the directory name
};
private:
Dirname manglePathname(const std::string &pathname);
std::chrono::steady_clock::time_point m_start_time; //!< Start the clock
unsigned int m_total_dirs; //!< Count how many have been processed
bool m_is_json; //!< Display results in JSON format for debugging
OracleDbConn m_castordb; //!< Oracle database for CASTOR->CTA migration tables
unsigned int m_batch_size; //!< Number of records to fetch from the DB at a time
unsigned int m_min_depth; //!< Minimum depth of directory tree to import
unsigned int m_max_depth; //!< Maximum depth of directory tree to import
unsigned int m_cur_depth; //!< Current depth imported
std::string m_castor_prefix; //!< CASTOR namespace prefix to strip
std::map<unsigned int, std::string> m_storageClass; //!< Mapping of CASTOR file class IDs to CTA storage classes
};
EosInsertMissingDirs::EosInsertMissingDirs(const std::string &configfile) :
m_start_time(std::chrono::steady_clock::now()),
m_total_dirs(0)
{
// Parse configuration file
XrdSsiPb::Config config(configfile);
auto dbconn = config.getOptionValueStr("castor.db_login");
auto is_json = config.getOptionValueBool("castor.json");
auto max_num_conns = config.getOptionValueInt("castor.max_num_connections");
auto batch_size = config.getOptionValueInt("castor.batch_size");
auto min_depth = config.getOptionValueInt("castor.min_depth");
auto max_depth = config.getOptionValueInt("castor.max_depth");
auto castor_prefix = config.getOptionValueStr("castor.prefix");
// Connect to Oracle
if(!dbconn.first) {
throw std::runtime_error("castor.db_login must be specified in the config file in the form oracle:user/password@TNS");
}
m_castordb.connect(dbconn.second, max_num_conns.first ? max_num_conns.second : 1);
// Set parameters and defaults
m_is_json = is_json.first ? is_json.second : false;
m_min_depth = min_depth.first ? min_depth.second : 1;
m_max_depth = max_depth.first ? max_depth.second : 99999;
m_batch_size = batch_size.first ? batch_size.second : 10000;
m_castor_prefix = castor_prefix.first ? castor_prefix.second : "/castor/cern.ch";
m_cur_depth = m_min_depth;
if(m_min_depth > m_max_depth) {
throw std::runtime_error("Configuration error: min_depth cannot be greater than max_depth!");
}
}
void EosInsertMissingDirs::process()
{
std::set<std::string> existingDirs;
std::map<std::string, int> missingDirs;
// If result set is empty, get next results
std::cerr << "Reading directory entries from CASTOR DB...";
m_castordb.query("SELECT PATH FROM CTADIRSHELPER");
while(true) {
existingDirs.insert(m_castordb.getResultColumnString("PATH"));
if(!m_castordb.nextRow()) break;
}
std::cerr << "(" << existingDirs.size() << ")...done." << std::endl;
std::cerr << "Creating list of directories to insert...";
for(auto dir : existingDirs) {
for(int n = dir.find_last_of('/'); n != 0; n = dir.find_last_of('/')) {
dir.resize(n);
if(existingDirs.count(dir) == 0 && missingDirs.count(dir) == 0) {
int depth = 2; // for /castor/cern.ch which has been stripped off
for(auto &c : dir) { if(c == '/') ++depth; }
missingDirs[dir] = depth;
++m_total_dirs;
}
}
}
std::cerr << "(" << missingDirs.size() << ")...done." << std::endl;
std::cerr << "Inserting directories into DB...";
m_castordb.query("SELECT MAX(FILEID) AS MAX_FILE_ID FROM CTADIRSHELPER");
uint64_t fileid = m_castordb.getResultColumnUint64("MAX_FILE_ID");
std::cerr << "(" << std::to_string(fileid) << ")...";
int cnt = 0;
for(auto &m : missingDirs) {
m_castordb.execute("INSERT INTO CTADIRSHELPER(FILEID, PATH, DISK_UID, DISK_GID, FILEMODE, CTIME, MTIME, CLASSID, PARENT_FILEID, DEPTH) "
"VALUES(" + std::to_string(++fileid) + ",\'" + m.first + "\'," +
"22103,1028,16877,1543833247,1543833248,5012,0," + std::to_string(m.second) + ")");
if(++cnt > 1000) { cnt = 0; std::cerr << "."; }
}
std::cerr << "done." << std::endl;
auto elapsed_time = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::steady_clock::now() - m_start_time);
std::cerr << "Added " << m_total_dirs << " directories in " << elapsed_time.count() << "s" << std::endl;
}
EosInsertMissingDirs::Dirname EosInsertMissingDirs::manglePathname(const std::string &pathname)
{
Dirname dir;
size_t clip = (pathname.find(m_castor_prefix) == 0) ? m_castor_prefix.length() : 0;
dir.pathname = pathname.substr(clip);
clip = dir.pathname.find_last_of('/');
dir.basename = (clip == std::string::npos) ? dir.pathname : dir.pathname.substr(clip+1);
return dir;
}
}} // namespace cta::migration
void throwUsage(const std::string &program, const std::string &error_txt)
{
std::stringstream help;
help << program << ": " << error_txt << std::endl
<< "Usage: " << program << " [--config <config_file>]";
throw std::runtime_error(help.str());
}
int main(int argc, const char* argv[])
{
std::string configfile = "/etc/cta/castor-migration.conf";
try {
for(auto i = 1; i < argc; ++i) {
std::string option(argv[i]);
if(option == "--config" && argc > ++i) {
configfile = argv[i];
continue;
}
throwUsage(argv[0], "invalid option " + option);
}
cta::migration::EosInsertMissingDirs insertMissingDirs(configfile);
insertMissingDirs.process();
} catch(std::runtime_error &ex) {
std::cerr << ex.what() << std::endl;
return -1;
}
return 0;
}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

imprint | privacy | Conditions of Use