Commit cc8e8534 authored by Michael Davis's avatar Michael Davis
Browse files

[migration] Adds --check and --skip n options

parent ec26cba1
......@@ -32,10 +32,12 @@ namespace migration {
std::exception_ptr EosImportFiles::m_exptr = nullptr;
EosImportFiles::EosImportFiles(const std::string &configfile, std::string storageClass, bool retry) :
EosImportFiles::EosImportFiles(const std::string &configfile, std::string storageClass, bool retry, bool check, uint64_t skip) :
m_start_time(std::chrono::steady_clock::now()),
m_total_files(0),
m_is_retry(retry),
m_is_check(check),
m_skip(skip),
m_storageClassFilter(storageClass)
{
// Parse configuration file
......
......@@ -28,7 +28,7 @@ namespace migration {
class EosImportFiles
{
public:
EosImportFiles(const std::string &configfile, std::string storageClass = "", bool retry = false);
EosImportFiles(const std::string &configfile, std::string storageClass = "", bool retry = false, bool check = false, uint64_t skip = 0);
void preFlightCheck();
void getStorageClasses();
......@@ -54,6 +54,8 @@ private:
bool m_is_dry_run; //!< Pull the data from Oracle but don't inject into EOS
bool m_is_retry; //!< We are retrying failed jobs
bool m_is_retry_strict; //!< Require all metadata to match during retries
bool m_is_check; //!< Check if each file exists in EOS namespace before injection
uint64_t m_skip; //!< Number of DB rows to skip before starting injection
std::string m_storageClassFilter; //!< File class/storage class to filter on
std::string m_tableName; //!< Table containing list of files
std::unique_ptr<rdbms::ConnPool> m_dbConnPool; //!< Pool of Oracle database connections
......
......@@ -38,17 +38,23 @@ struct EosImportFilesCmdLine {
std::string storageClass;
Command cmd;
bool retry;
bool check;
uint64_t skip;
EosImportFilesCmdLine() : retry(false) {}
EosImportFilesCmdLine() : retry(false), check(false), skip(0) {}
static void printUsage(std::ostream &os) {
os <<
"Usage:" << std::endl <<
" eos-import-files [--config <configuration_path>] [--storage-class <storage_class>|--help|--failure-modes|--list-failures|--retry|--forget-failures]" << std::endl <<
" eos-import-files [--config <configuration_path>][--check][--skip <n>] [--help|--storage-class <storage_class>|--failure-modes|--list-failures|--retry|--forget-failures]" << std::endl <<
"Options:" << std::endl <<
" -c, --config <configuration_path>" << std::endl <<
" The optional path to the configuration file." << std::endl <<
" The default value if not set is " << s_defaultConfigPath << std::endl <<
" -x, --check" << std::endl <<
" Check if each file exists in the EOS namespace before injecting (slower, but useful when resuming a failed import)" << std::endl <<
" -k, --skip <n>" << std::endl <<
" Skip the first n files before resuming import (useful when resuming a failed import)" << std::endl <<
" -f, --failure-modes" << std::endl <<
" List the distinct error messages for insertions which failed." << std::endl <<
" -F, --forget-failures" << std::endl <<
......@@ -68,6 +74,8 @@ struct EosImportFilesCmdLine {
static struct option longopts[] = {
{ "config", required_argument, NULL, 'c' },
{ "check", no_argument, NULL, 'x' },
{ "skip", required_argument, NULL, 'k' },
{ "storage-class", required_argument, NULL, 's' },
{ "failure-modes", no_argument, NULL, 'f' },
{ "help", no_argument, NULL, 'h' },
......@@ -83,7 +91,7 @@ struct EosImportFilesCmdLine {
cmdLine.cmd = INJECT;
int opt = 0;
while ((opt = getopt_long(argc, argv, "c:s:fhlrF", longopts, NULL)) != -1) {
while ((opt = getopt_long(argc, argv, "c:k:s:xfhlrF", longopts, NULL)) != -1) {
switch (opt) {
case 'c':
cmdLine.configPath = optarg;
......@@ -111,6 +119,14 @@ struct EosImportFilesCmdLine {
if(cmdLine.cmd != INJECT) throw std::runtime_error("Invalid combination of options");
cmdLine.cmd = FORGET_FAILURES;
break;
case 'k':
if(cmdLine.cmd != INJECT) throw std::runtime_error("Invalid combination of options");
cmdLine.skip = strtoul(optarg, NULL, 0);
break;
case 'x':
if(cmdLine.cmd != INJECT) throw std::runtime_error("Invalid combination of options");
cmdLine.check = true;
break;
case ':': // Missing parameter
{
std::ostringstream msg;
......@@ -199,7 +215,7 @@ int main(const int argc, char ** argv)
return 0;
}
case EosImportFilesCmdLine::INJECT: {
cta::migration::EosImportFiles importFiles(cmdLine.configPath, cmdLine.storageClass, cmdLine.retry);
cta::migration::EosImportFiles importFiles(cmdLine.configPath, cmdLine.storageClass, cmdLine.retry, cmdLine.check, cmdLine.skip);
importFiles.preFlightCheck();
importFiles.getStorageClasses();
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment