Commit 6c99dea0 authored by Michael Davis's avatar Michael Davis
Browse files

[migration] Adds test directory insert program

parent dbba3a57
......@@ -108,8 +108,8 @@ EosImportDirs::EosImportDirs(const std::string &configfile) :
m_min_depth = min_depth.first ? min_depth.second : 1;
m_max_depth = max_depth.first ? max_depth.second : 99999;
m_batch_size = batch_size.first ? batch_size.second : 10000;
m_castor_prefix = castor_prefix.first ? castor_prefix.second : "/castor/cern.ch/";
m_eos_prefix = eos_prefix.first ? eos_prefix.second : "/eos/grpc/";
m_castor_prefix = castor_prefix.first ? castor_prefix.second : "/castor/cern.ch";
m_eos_prefix = eos_prefix.first ? eos_prefix.second : "/eos/grpc";
m_cur_depth = m_min_depth;
if(m_min_depth > m_max_depth) {
......
......@@ -96,8 +96,8 @@ EosImportFiles::EosImportFiles(const std::string &configfile) :
m_is_json = is_json.first ? is_json.second : false;
m_is_dry_run = is_dry_run.first ? is_dry_run.second : false;
m_batch_size = batch_size.first ? batch_size.second : 10000;
m_castor_prefix = castor_prefix.first ? castor_prefix.second : "/castor/cern.ch/";
m_eos_prefix = eos_prefix.first ? eos_prefix.second : "/eos/grpc/";
m_castor_prefix = castor_prefix.first ? castor_prefix.second : "/castor/cern.ch";
m_eos_prefix = eos_prefix.first ? eos_prefix.second : "/eos/grpc";
}
......
......@@ -19,6 +19,7 @@
#include <iostream>
#include <sstream>
#include <XrdSsiPbConfig.hpp>
#include <XrdSsiPbLog.hpp>
#include "GrpcClient.hpp"
......@@ -27,75 +28,124 @@
namespace cta {
namespace migration {
class EosTestDirInject
{
public:
EosTestDirInject();
void process(const std::string &pathname, uint64_t fileId);
EosTestDirInject(const std::string &configfile);
void inject(const std::string &path, uint64_t fileid);
std::string ping(const std::string &payload) {
return m_eosgrpc->ping(payload);
}
struct Dirname {
std::string basename; //!< Just the directory name
std::string pathname; //!< The full path including the directory name
};
private:
std::string getBasename(const std::string &pathname);
Dirname manglePathname(const std::string &pathname);
std::chrono::steady_clock::time_point m_start_time; //!< Start the clock
bool m_is_json; //!< Display results in JSON format for debugging
std::unique_ptr<eos::client::GrpcClient> m_eosgrpc; //!< EOS gRPC API interface
std::string m_castor_prefix; //!< CASTOR namespace prefix to strip
std::string m_eos_prefix; //!< EOS namespace prefix to prepend
std::map<unsigned int, std::string> m_storageClass; //!< Mapping of CASTOR file class IDs to CTA storage classes
};
EosTestDirInject::EosTestDirInject() :
m_is_json(true)
EosTestDirInject::EosTestDirInject(const std::string &configfile) :
m_start_time(std::chrono::steady_clock::now())
{
std::string endpoint("ctadevmichael.cern.ch:50051");
std::string token("helloworld");
// Parse configuration file
XrdSsiPb::Config config(configfile);
auto is_json = config.getOptionValueBool("castor.json");
auto castor_prefix = config.getOptionValueStr("castor.prefix");
auto eos_prefix = config.getOptionValueStr("eos.prefix");
auto endpoint = config.getOptionValueStr("eos.endpoint");
auto token = config.getOptionValueStr("eos.token");
// Connect to EOS
m_eosgrpc = eos::client::GrpcClient::Create(endpoint, token);
m_eosgrpc = eos::client::GrpcClient::Create(endpoint.first ? endpoint.second : "localhost:50051", token.second);
// Set parameters and defaults
m_is_json = is_json.first ? is_json.second : false;
m_castor_prefix = castor_prefix.first ? castor_prefix.second : "/castor/cern.ch";
m_eos_prefix = eos_prefix.first ? eos_prefix.second : "/eos/grpc";
}
void EosTestDirInject::process(const std::string &pathname, uint64_t fileId)
void EosTestDirInject::inject(const std::string &path, uint64_t fileid)
{
eos::rpc::ContainerMdProto dir;
dir.set_id(fileId);
//dir.set_parent_id(m_castordb.getResultColumnUint64("PARENT_FILEID"));
dir.set_id(fileid);
dir.set_parent_id(12345);
dir.set_uid(1000);
dir.set_gid(1001);
dir.set_gid(1000);
// we don't care about dir.tree_size
dir.set_mode(0755);
// we don't care about dir.flags
// CTIME is missing from intermediate table
dir.mutable_ctime()->set_sec(1563362695);
dir.mutable_mtime()->set_sec(1563362696);
dir.mutable_stime()->set_sec(1563362697);
// Timestamps
dir.mutable_ctime()->set_sec(1553900400);
dir.mutable_mtime()->set_sec(1553900400);
// we don't care about dir.stime (sync time, used for CERNBox)
// BTIME is set as an extended attribute (see below)
// Directory name and full path
dir.set_name(getBasename(pathname));
dir.set_path(pathname);
auto dirname = manglePathname(path);
dir.set_name(dirname.basename);
dir.set_path(dirname.pathname);
// Extended attributes:
//
// 1. Storage Class
dir.mutable_xattrs()->insert(google::protobuf::MapPair<std::string,std::string>("CTA_StorageClass", "my_storage_class"));
// 2. Birth Time
// POSIX ATIME (Access Time) is used by CASTOR to store the file creation time. EOS calls this "birth time",
// but there is no place in the namespace to store it, so it is stored as an extended attribute.
dir.mutable_xattrs()->insert(google::protobuf::MapPair<std::string,std::string>("eos.btime", "1553900400"));
// Extended attributes
dir.mutable_xattrs()->insert(google::protobuf::MapPair<std::string,std::string>("CTA_StorageClass", "MyStorageClass"));
std::vector<eos::rpc::ContainerMdProto> dirs;
dirs.push_back(dir);
// Put results on stdout for debugging
if(m_is_json) {
std::cout << '[' << XrdSsiPb::Log::DumpProtobuf(&dir) << ']' << std::endl;
char delim = '[';
for(auto &dir : dirs) {
std::cout << delim << XrdSsiPb::Log::DumpProtobuf(&dir);
delim = ',';
}
if(delim == ',') std::cout << "]";
}
// Inject directory into EOS
std::vector<eos::rpc::ContainerMdProto> dirs;
dirs.push_back(dir);
int retc = m_eosgrpc->ContainerInsert(dirs);
if(retc != 0) {
throw std::runtime_error("EosTestDirInject::processBatch(): ContainerInsert failed with error " + std::to_string(retc));
}
// Inject directories into EOS
int retc = m_eosgrpc->ContainerInsert(dirs);
if(retc != 0) {
throw std::runtime_error("EosTestDirInject::processBatch(): ContainerInsert failed with error " + std::to_string(retc));
}
auto elapsed_time = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::steady_clock::now() - m_start_time);
std::cerr << "Processed 1 directory in " << elapsed_time.count() << "s" << std::endl;
}
std::string EosTestDirInject::getBasename(const std::string &pathname)
EosTestDirInject::Dirname EosTestDirInject::manglePathname(const std::string &pathname)
{
auto clip = pathname.find_last_of('/');
return (clip == std::string::npos) ? pathname : pathname.substr(clip+1);
Dirname dir;
size_t clip = (pathname.find(m_castor_prefix) == 0) ? m_castor_prefix.length() : 0;
dir.pathname = m_eos_prefix + pathname.substr(clip);
clip = dir.pathname.find_last_of('/');
dir.basename = (clip == std::string::npos) ? dir.pathname : dir.pathname.substr(clip+1);
return dir;
}
}} // namespace cta::migration
......@@ -106,7 +156,7 @@ void throwUsage(const std::string &program, const std::string &error_txt)
std::stringstream help;
help << program << ": " << error_txt << std::endl
<< "Usage: " << program << " [--pathname <pathname>] [--set-file-id <hex_file_id>]";
<< "Usage: " << program << " [--config <config_file>] [--fileid <fileid>] ping|--path <path>";
throw std::runtime_error(help.str());
}
......@@ -114,24 +164,41 @@ void throwUsage(const std::string &program, const std::string &error_txt)
int main(int argc, const char* argv[])
{
std::string pathname("/eos/grpc/test0");
uint64_t fileId = 0;
std::string configfile = "/etc/cta/castor-migration.conf";
std::string path;
uint64_t fileid = 0;
bool doPing = false;
try {
for(auto i = 1; i < argc; ++i) {
std::string option(argv[i]);
if(option == "--pathname" && argc > ++i) {
pathname = argv[i];
if(option == "--config" && argc > ++i) {
configfile = argv[i];
continue;
} else if(option == "--path" && argc > ++i) {
path = argv[i];
continue;
} else if(option == "--set-file-id" && argc > ++i) {
fileId = strtoul(argv[i], NULL, 16);
} else if(option == "--fileid" && argc > ++i) {
fileid = strtoul(argv[i], NULL, 0);
continue;
} else if(option == "ping") {
doPing = true;
continue;
}
throwUsage(argv[0], "invalid option " + option);
}
cta::migration::EosTestDirInject injectDirs;
injectDirs.process(pathname, fileId);
cta::migration::EosTestDirInject testDirInject(configfile);
if(doPing) {
std::cout << "Pinging EOS MGM using gRPC API..." << std::endl;
auto pingStr = testDirInject.ping("Ping from EosTestDirInject");
std::cout << "Ping successful. Server responded with: " << pingStr << std::endl;
} else {
if(path.empty()) throwUsage(argv[0], "path not specified");
testDirInject.inject(path, fileid);
}
} catch(std::runtime_error &ex) {
std::cerr << ex.what() << std::endl;
return -1;
......
#!/bin/sh
TEST_HOME=~/CTA_build/migration/gRPC
INJECT=${TEST_HOME}/eos-test-dir-inject
# Create directory with system-assigned file id
${INJECT} --path /castor/cern.ch/my_test_dir | json-pretty-print.sh
eos ls -l /eos/grpc
eos fileinfo /eos/grpc/my_test_dir
eos rmdir /eos/grpc/my_test_dir
# Create directory with self-assigned file id
${INJECT} --fileid 9876543210 --path /castor/cern.ch/my_test_dir | json-pretty-print.sh
eos fileinfo /eos/grpc/my_test_dir
# Try again -- should fail
${INJECT} --fileid 9876543210 --path /castor/cern.ch/my_test_dir >/dev/null
eos fileinfo /eos/grpc/my_test_dir
# Remove and try again
eos rmdir /eos/grpc/my_test_dir
sudo systemctl restart eos@*
sleep 3
${INJECT} --fileid 9876543210 --path /castor/cern.ch/my_test_dir >/dev/null
eos fileinfo /eos/grpc/my_test_dir
eos rmdir /eos/grpc/my_test_dir
eos ls /eos/grpc
sudo systemctl restart eos@*
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment