Commit eba07433 authored by Michael Davis's avatar Michael Davis
Browse files

[migration] Adds --storage-class filter to eos-import-files

parent 5c1d22db
......@@ -27,10 +27,11 @@
namespace cta {
namespace migration {
EosImportFiles::EosImportFiles(const std::string &configfile, bool retry) :
EosImportFiles::EosImportFiles(const std::string &configfile, std::string storageClass, bool retry) :
m_start_time(std::chrono::steady_clock::now()),
m_total_files(0),
m_is_retry(retry)
m_is_retry(retry),
m_storageClassFilter(storageClass)
{
// Parse configuration file
XrdSsiPb::Config config(configfile);
......@@ -154,7 +155,7 @@ void EosImportFiles::getStorageClasses()
void EosImportFiles::getFileMetadata()
{
const std::string sql =
std::string sql =
"SELECT "
" A.FILEID,"
" A.PARENT_FILEID,"
......@@ -176,7 +177,29 @@ void EosImportFiles::getFileMetadata()
" A.S_MTIME "
"FROM " + m_tableName + " A, CTADIRSHELPER B "
"WHERE A.PARENT_FILEID = B.FILEID";
m_selectFilesDbStmt = m_dbConn.createStmt(sql);
if(m_storageClassFilter.empty()) {
m_selectFilesDbStmt = m_dbConn.createStmt(sql);
} else {
// Filter by storage class if this option was provided
std::string sc;
try {
// Try looking up the string value in the list of known storage classes
sc = m_storageClassRev.at(m_storageClassFilter);
} catch(...) {
// Assume that the numeric ID was provided
sc = m_storageClassFilter;
}
auto scid = strtoul(sc.c_str(), NULL, 0);
if(scid == 0) {
throw std::runtime_error("Could not resolve \"" + m_storageClassFilter + "\" to a valid storage class");
}
sql += " AND A.CLASSID = :CLASSID";
m_selectFilesDbStmt = m_dbConn.createStmt(sql);
m_selectFilesDbStmt.bindUint64(":CLASSID", scid);
}
std::cerr << "Executing SELECT query against " << m_tableName << " table...";
m_selectFilesDbRset = m_selectFilesDbStmt.executeQuery();
std::cerr << "done." << std::endl;
......
......@@ -27,7 +27,7 @@ namespace migration {
class EosImportFiles
{
public:
EosImportFiles(const std::string &configfile, bool retry = false);
EosImportFiles(const std::string &configfile, std::string storageClass = "", bool retry = false);
void getStorageClasses();
void listFailureModes();
......@@ -49,6 +49,7 @@ private:
bool m_is_json; //!< Display results in JSON format for debugging
bool m_is_dry_run; //!< Pull the data from Oracle but don't inject into EOS
bool m_is_retry; //!< We are retrying failed jobs
std::string m_storageClassFilter; //!< File class/storage class to filter on
std::string m_tableName; //!< Table containing list of files
std::unique_ptr<rdbms::ConnPool> m_dbConnPool; //!< Pool of Oracle database connections
rdbms::Conn m_dbConn; //!< The current database connection
......
......@@ -35,6 +35,7 @@ struct EosImportFilesCmdLine {
static const std::string s_defaultConfigPath;
std::string configPath;
std::string storageClass;
Command cmd;
bool retry;
......@@ -43,21 +44,23 @@ struct EosImportFilesCmdLine {
static void printUsage(std::ostream &os) {
os <<
"Usage:" << std::endl <<
" eos-import-files [--config <configuration_path>] [--help|--failure-modes|--list-failures|--retry|--forget-failures]" << std::endl <<
" eos-import-files [--config <configuration_path>] [--storage-class <storage_class>|--help|--failure-modes|--list-failures|--retry|--forget-failures]" << std::endl <<
"Options:" << std::endl <<
" -c, --config configuration_path" << std::endl <<
" -c, --config <configuration_path>" << std::endl <<
" The optional path to the configuration file." << std::endl <<
" The default value if not set is " << s_defaultConfigPath << std::endl <<
" -f, --failure-modes" << std::endl <<
" List the distinct error messages for insertions which failed." << std::endl <<
" -F, --forget-failures" << std::endl <<
" Remove all paths under <top_level_path> from CTADIRSFAILED." << std::endl <<
" -h, --help" << std::endl <<
" Prints this usage message." << std::endl <<
" -l, --list-failures" << std::endl <<
" Detailed listing of the directory insertions which failed." << std::endl <<
" -r, --retry" << std::endl <<
" Retry failed directory injections. (i.e. take input from CTADIRSFAILED instead of CTADIRSHELPER)" << std::endl <<
" -F, --forget-failures" << std::endl <<
" Remove all paths under <top_level_path> from CTADIRSFAILED." << std::endl;
" -s, --storage-class <storage_class>" << std::endl <<
" Filter the input to the specified storage class. Storage class can be the string name or numeric id." << std::endl;
}
static EosImportFilesCmdLine parseCmdLine(const int argc, char **argv) {
......@@ -65,6 +68,7 @@ struct EosImportFilesCmdLine {
static struct option longopts[] = {
{ "config", required_argument, NULL, 'c' },
{ "storage-class", required_argument, NULL, 's' },
{ "failure-modes", no_argument, NULL, 'f' },
{ "help", no_argument, NULL, 'h' },
{ "list-failures", no_argument, NULL, 'l' },
......@@ -79,11 +83,14 @@ struct EosImportFilesCmdLine {
cmdLine.cmd = INJECT;
int opt = 0;
while ((opt = getopt_long(argc, argv, "c:fhlrF", longopts, NULL)) != -1) {
while ((opt = getopt_long(argc, argv, "c:s:fhlrF", longopts, NULL)) != -1) {
switch (opt) {
case 'c':
cmdLine.configPath = optarg;
break;
case 's':
cmdLine.storageClass = optarg;
break;
case 'r':
if(cmdLine.cmd != INJECT) throw std::runtime_error("Invalid combination of options");
cmdLine.retry = true;
......@@ -128,6 +135,10 @@ struct EosImportFilesCmdLine {
} // switch(opt)
} // while getopt_long()
if(cmdLine.cmd != INJECT && !cmdLine.storageClass.empty()) {
throw std::runtime_error("Invalid combination of options");
}
if(cmdLine.configPath.empty()) {
cmdLine.configPath = s_defaultConfigPath;
}
......@@ -188,7 +199,7 @@ int main(const int argc, char ** argv)
return 0;
}
case EosImportFilesCmdLine::INJECT: {
cta::migration::EosImportFiles importFiles(cmdLine.configPath, cmdLine.retry);
cta::migration::EosImportFiles importFiles(cmdLine.configPath, cmdLine.storageClass, cmdLine.retry);
importFiles.getStorageClasses();
importFiles.getFileMetadata();
while(importFiles.processBatch()) ;
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment