Commit 44fd91ba authored by Michael Davis's avatar Michael Davis
Browse files

[migration] Adds eos-test-file-inject to cta-migration RPM

parent f067fea6
......@@ -308,6 +308,7 @@ directory metadata into the EOS namespace.
%attr(0755,root,root) %{_bindir}/eos-import-dirs
%attr(0755,root,root) %{_bindir}/eos-import-files
%attr(0755,root,root) %{_bindir}/eos-test-dir-inject
%attr(0755,root,root) %{_bindir}/eos-test-file-inject
%attr(0755,root,root) %{_bindir}/eos-test-inject.sh
%package -n cta-rmcd
......
......@@ -32,12 +32,17 @@ add_executable(eos-import-files EosImportFiles.cpp GrpcClient.cpp)
target_link_libraries(eos-import-files EosMigration ${PROTOBUF3_LIBRARIES} ${GRPC_LIBRARY} ${GRPC_GRPC++_LIBRARY} ctacatalogue)
set_property(TARGET eos-import-files APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH})
# Standalone test tool
# Standalone test tools
add_executable(eos-test-dir-inject EosTestDirInject.cpp GrpcClient.cpp)
target_link_libraries(eos-test-dir-inject EosMigration ${PROTOBUF3_LIBRARIES} ${GRPC_LIBRARY} ${GRPC_GRPC++_LIBRARY} ctacatalogue)
set_property(TARGET eos-test-dir-inject APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH})
add_executable(eos-test-file-inject EosTestFileInject.cpp GrpcClient.cpp)
target_link_libraries(eos-test-file-inject EosMigration ${PROTOBUF3_LIBRARIES} ${GRPC_LIBRARY} ${GRPC_GRPC++_LIBRARY} ctacatalogue)
set_property(TARGET eos-test-file-inject APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH})
install(TARGETS eos-import-dirs DESTINATION usr/bin)
install(TARGETS eos-import-files DESTINATION usr/bin)
install(TARGETS eos-test-dir-inject DESTINATION usr/bin)
install(TARGETS eos-test-file-inject DESTINATION usr/bin)
install(FILES ${CMAKE_SOURCE_DIR}/migration/gRPC/eos-test-inject.sh DESTINATION usr/bin)
......@@ -126,10 +126,10 @@ void EosTestDirInject::inject(const std::string &path, uint64_t fileid)
}
// Inject directories into EOS
int retc = m_eosgrpc->ContainerInsert(dirs);
if(retc != 0) {
throw std::runtime_error("EosTestDirInject::processBatch(): ContainerInsert failed with error " + std::to_string(retc));
}
int retc = m_eosgrpc->ContainerInsert(dirs);
if(retc != 0) {
throw std::runtime_error("EosTestDirInject::inject(): ContainerInsert failed with error " + std::to_string(retc));
}
auto elapsed_time = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::steady_clock::now() - m_start_time);
std::cerr << "Processed 1 directory in " << elapsed_time.count() << "s" << std::endl;
......
/*!
* @project The CERN Tape Archive (CTA)
* @brief Inject directory metadata into EOS using gRPC
* @copyright Copyright 2019 CERN
* @license This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <iostream>
#include <sstream>
#include <XrdSsiPbConfig.hpp>
#include <XrdSsiPbLog.hpp>
#include "GrpcClient.hpp"
namespace cta {
namespace migration {
class EosTestFileInject
{
public:
EosTestFileInject(const std::string &configfile);
void inject(const std::string &path, uint64_t fileid);
std::string ping(const std::string &payload) {
return m_eosgrpc->ping(payload);
}
struct Dirname {
std::string basename; //!< Just the directory name
std::string pathname; //!< The full path including the directory name
};
private:
static std::string convertChecksum(uint32_t adler32);
Dirname manglePathname(const std::string &pathname);
std::chrono::steady_clock::time_point m_start_time; //!< Start the clock
bool m_is_json; //!< Display results in JSON format for debugging
std::unique_ptr<eos::client::GrpcClient> m_eosgrpc; //!< EOS gRPC API interface
std::string m_castor_prefix; //!< CASTOR namespace prefix to strip
std::string m_eos_prefix; //!< EOS namespace prefix to prepend
std::map<unsigned int, std::string> m_storageClass; //!< Mapping of CASTOR file class IDs to CTA storage classes
};
EosTestFileInject::EosTestFileInject(const std::string &configfile) :
m_start_time(std::chrono::steady_clock::now())
{
// Parse configuration file
XrdSsiPb::Config config(configfile);
auto is_json = config.getOptionValueBool("castor.json");
auto castor_prefix = config.getOptionValueStr("castor.prefix");
auto eos_prefix = config.getOptionValueStr("eos.prefix");
auto endpoint = config.getOptionValueStr("eos.endpoint");
auto token = config.getOptionValueStr("eos.token");
// Connect to EOS
m_eosgrpc = eos::client::GrpcClient::Create(endpoint.first ? endpoint.second : "localhost:50051", token.second);
// Set parameters and defaults
m_is_json = is_json.first ? is_json.second : false;
m_castor_prefix = castor_prefix.first ? castor_prefix.second : "/castor/cern.ch";
m_eos_prefix = eos_prefix.first ? eos_prefix.second : "/eos/grpc";
}
void EosTestFileInject::inject(const std::string &path, uint64_t fileid)
{
// See EOS common/LayoutId.hh for definitions of constants
const int kAdler = 0x2;
const int kReplica = (0x1 << 4);
const int kStripeSize = (0x0 << 8); // 1 stripe
const int kStripeWidth = (0x0 << 16); // 4K blocks
const int kBlockChecksum = (0x1 << 20);
// Layout id should be 00100012
const uint64_t layout = kReplica | kAdler | kStripeSize | kStripeWidth | kBlockChecksum;
eos::rpc::FileMdProto file;
file.set_id(fileid);
file.set_cont_id(12345);
file.set_uid(1000);
file.set_gid(1000);
file.set_size(42);
file.set_layout_id(layout);
file.set_flags(0755);
file.mutable_checksum()->set_type("adler");
file.mutable_checksum()->set_value(convertChecksum(0xaabbccdd));
// Timestamps
file.mutable_ctime()->set_sec(1553900400);
file.mutable_mtime()->set_sec(1553900400);
// we don't care about file.stime (sync time, used for CERNBox)
// BTIME is set as an extended attribute (see below)
// Filename and path
file.set_name(path);
file.set_path(m_eos_prefix + "/" + file.name());
// we don't care about link_name
// Extended attributes:
//
// 1. Archive File ID
std::string archiveId(std::to_string(file.id()));
file.mutable_xattrs()->insert(google::protobuf::MapPair<std::string,std::string>("CTA_ArchiveFileId", archiveId));
// 2. Storage Class
file.mutable_xattrs()->insert(google::protobuf::MapPair<std::string,std::string>("CTA_StorageClass", "my_storage_class"));
// 3. Birth Time
// POSIX ATIME (Access Time) is used by CASTOR to store the file creation time. EOS calls this "birth time",
// but there is no place in the namespace to store it, so it is stored as an extended attribute.
file.mutable_xattrs()->insert(google::protobuf::MapPair<std::string,std::string>("eos.btime", "1553900400"));
// Indicate that there is a tape-resident replica of this file
file.mutable_locations()->Add(65535);
// we don't care about unlink_locations (placeholder for files scheduled for deletion)
std::vector<eos::rpc::FileMdProto> files;
files.push_back(file);
// Put results on stdout for debugging
if(m_is_json) {
char delim = '[';
for(auto &file : files) {
std::cout << delim << XrdSsiPb::Log::DumpProtobuf(&file);
delim = ',';
}
if(delim == ',') std::cout << "]";
}
// Inject file into EOS
int retc = m_eosgrpc->FileInsert(files);
if(retc != 0) {
throw std::runtime_error("EosTestFileInject::inject(): FileInsert failed with error " + std::to_string(retc));
}
auto elapsed_time = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::steady_clock::now() - m_start_time);
std::cerr << "Processed 1 file in " << elapsed_time.count() << "s" << std::endl;
}
std::string EosTestFileInject::convertChecksum(uint32_t adler32)
{
char bytes[4];
for(int i = 3; i >= 0; --i, adler32 >>= 8) { bytes[i] = adler32 & 0xFF; }
return std::string(bytes, 4);
}
EosTestFileInject::Dirname EosTestFileInject::manglePathname(const std::string &pathname)
{
Dirname dir;
size_t clip = (pathname.find(m_castor_prefix) == 0) ? m_castor_prefix.length() : 0;
dir.pathname = m_eos_prefix + pathname.substr(clip);
clip = dir.pathname.find_last_of('/');
dir.basename = (clip == std::string::npos) ? dir.pathname : dir.pathname.substr(clip+1);
return dir;
}
}} // namespace cta::migration
void throwUsage(const std::string &program, const std::string &error_txt)
{
std::stringstream help;
help << program << ": " << error_txt << std::endl
<< "Usage: " << program << " [--config <config_file>] [--fileid <fileid>] ping|--path <path>";
throw std::runtime_error(help.str());
}
int main(int argc, const char* argv[])
{
std::string configfile = "/etc/cta/castor-migration.conf";
std::string path;
uint64_t fileid = 0;
bool doPing = false;
try {
for(auto i = 1; i < argc; ++i) {
std::string option(argv[i]);
if(option == "--config" && argc > ++i) {
configfile = argv[i];
continue;
} else if(option == "--path" && argc > ++i) {
path = argv[i];
continue;
} else if(option == "--fileid" && argc > ++i) {
fileid = strtoul(argv[i], NULL, 0);
continue;
} else if(option == "ping") {
doPing = true;
continue;
}
throwUsage(argv[0], "invalid option " + option);
}
cta::migration::EosTestFileInject testDirInject(configfile);
if(doPing) {
std::cout << "Pinging EOS MGM using gRPC API..." << std::endl;
auto pingStr = testDirInject.ping("Ping from EosTestFileInject");
std::cout << "Ping successful. Server responded with: " << pingStr << std::endl;
} else {
if(path.empty()) throwUsage(argv[0], "path not specified");
testDirInject.inject(path, fileid);
}
} catch(std::runtime_error &ex) {
std::cerr << ex.what() << std::endl;
return -1;
}
return 0;
}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment