Commit b0c03c31 authored by Michael Davis's avatar Michael Davis
Browse files

[migration] Enforces absolute pathnames for EOS dir/file injection

Now we are also tolerant of whether paths in config file do or do
not have a trailing slash.

Pathnames from the DB must have a leading slash if we have to strip off
the CASTOR prefix, but in any case we will convert all filenames to
absolute filenames by adding the EOS prefix.
parent 3f27376a
......@@ -310,7 +310,6 @@ directory metadata into the EOS namespace.
%attr(0755,root,root) %{_bindir}/eos-import-files
%attr(0755,root,root) %{_bindir}/eos-test-dir-inject
%attr(0755,root,root) %{_bindir}/eos-test-file-inject
%attr(0755,root,root) %{_bindir}/eos-insert-missing-dirs
%attr(0755,root,root) %{_bindir}/json-pretty-print.sh
%attr(0755,root,root) %{_bindir}/startvoexport.sh
%attr(0755,root,root) %{_bindir}/exporttapepool.sh
......
......@@ -23,29 +23,32 @@ include_directories(${XRD_SSI_PB_DIR}/include ${XROOTD_INCLUDE_DIR} ${XROOTD_INC
${PROTOBUF3_INCLUDE_DIRS})
# Directory import tool
add_executable(eos-import-dirs EosImportDirs.cpp GrpcClient.cpp)
add_executable(eos-import-dirs EosImportDirs.cpp GrpcClient.cpp GrpcUtils.cpp)
target_link_libraries(eos-import-dirs EosMigration ${PROTOBUF3_LIBRARIES} ${GRPC_LIBRARY} ${GRPC_GRPC++_LIBRARY} ctacatalogue)
set_property(TARGET eos-import-dirs APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH})
set_property(TARGET eos-import-dirs APPEND PROPERTY INSTALL_RPATH ${ORACLE-INSTANTCLIENT_RPATH})
# File import tool
add_executable(eos-import-files EosImportFiles.cpp GrpcClient.cpp)
add_executable(eos-import-files EosImportFiles.cpp GrpcClient.cpp GrpcUtils.cpp)
target_link_libraries(eos-import-files EosMigration ${PROTOBUF3_LIBRARIES} ${GRPC_LIBRARY} ${GRPC_GRPC++_LIBRARY} ctacatalogue)
set_property(TARGET eos-import-files APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH})
set_property(TARGET eos-import-files APPEND PROPERTY INSTALL_RPATH ${ORACLE-INSTANTCLIENT_RPATH})
## Tool to insert missing directories
add_executable(eos-insert-missing-dirs EosInsertMissingDirs.cpp)
target_link_libraries(eos-insert-missing-dirs ctacatalogue)
set_property(TARGET eos-insert-missing-dirs APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH})
set_property(TARGET eos-insert-missing-dirs APPEND PROPERTY INSTALL_RPATH ${ORACLE-INSTANTCLIENT_RPATH})
#add_executable(eos-insert-missing-dirs EosInsertMissingDirs.cpp)
#target_link_libraries(eos-insert-missing-dirs ctacatalogue)
#set_property(TARGET eos-insert-missing-dirs APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH})
#set_property(TARGET eos-insert-missing-dirs APPEND PROPERTY INSTALL_RPATH ${ORACLE-INSTANTCLIENT_RPATH})
# Standalone test tools
add_executable(eos-test-dir-inject EosTestDirInject.cpp GrpcClient.cpp)
#add_executable(grpc-utils-test GrpcUtilsTest.cpp GrpcUtils.cpp)
add_executable(eos-test-dir-inject EosTestDirInject.cpp GrpcClient.cpp GrpcUtils.cpp)
target_link_libraries(eos-test-dir-inject EosMigration ${PROTOBUF3_LIBRARIES} ${GRPC_LIBRARY} ${GRPC_GRPC++_LIBRARY})
set_property(TARGET eos-test-dir-inject APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH})
add_executable(eos-test-file-inject EosTestFileInject.cpp GrpcClient.cpp)
add_executable(eos-test-file-inject EosTestFileInject.cpp GrpcClient.cpp GrpcUtils.cpp)
target_link_libraries(eos-test-file-inject EosMigration ${PROTOBUF3_LIBRARIES} ${GRPC_LIBRARY} ${GRPC_GRPC++_LIBRARY})
set_property(TARGET eos-test-file-inject APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH})
......@@ -53,7 +56,6 @@ install(TARGETS eos-import-dirs
eos-import-files
eos-test-dir-inject
eos-test-file-inject
eos-insert-missing-dirs
DESTINATION usr/bin)
install(FILES ${CMAKE_SOURCE_DIR}/migration/gRPC/json-pretty-print.sh
DESTINATION usr/bin)
......
......@@ -27,6 +27,7 @@
#include <XrdSsiPbLog.hpp>
#include "GrpcClient.hpp"
#include "GrpcUtils.hpp"
#include "rdbms/Conn.hpp"
#include "rdbms/ConnPool.hpp"
#include "rdbms/Login.hpp"
......@@ -136,7 +137,7 @@ struct EosImportDirsCmdLine {
const std::string EosImportDirsCmdLine::s_defaultConfigPath("/etc/cta/castor-migration.conf");
class EosImportDirs
class EosImportDirs
{
public:
EosImportDirs(const std::string &configfile, const std::string &topLevelPath);
......@@ -145,14 +146,8 @@ public:
void getTreeDepth();
bool processBatch();
struct Dirname {
std::string basename; //!< Just the directory name
std::string pathname; //!< The full path including the directory name
};
private:
std::vector<eos::rpc::ContainerMdProto> getNextBatch();
Dirname manglePathname(const std::string &pathname);
std::chrono::steady_clock::time_point m_start_time; //!< Start the clock
unsigned int m_total_dirs; //!< Count how many have been processed
......@@ -232,8 +227,11 @@ EosImportDirs::EosImportDirs(const std::string &configfile, const std::string &t
m_is_json = is_json.first ? is_json.second : false;
m_is_dry_run = is_dry_run.first ? is_dry_run.second : false;
m_batch_size = batch_size.first ? batch_size.second : 10000;
m_castor_prefix = castor_prefix.first ? castor_prefix.second : "/castor/cern.ch";
m_eos_prefix = eos_prefix.first ? eos_prefix.second : "/eos/grpc";
m_castor_prefix = castor_prefix.first ? castor_prefix.second : "/castor/cern.ch/";
m_eos_prefix = eos_prefix.first ? eos_prefix.second : "/eos/grpc/";
// enforce a slash at beginning and end of prefixes
eos::client::checkPrefix(m_castor_prefix);
eos::client::checkPrefix(m_eos_prefix);
}
......@@ -311,7 +309,7 @@ std::vector<eos::rpc::ContainerMdProto> EosImportDirs::getNextBatch() {
// we don't care about dir.stime (sync time, used for CERNBox)
// Directory name and full path
auto dirname = manglePathname(m_selectDirsDbRset.columnString("PATH"));
auto dirname = eos::client::manglePathname(m_castor_prefix, m_eos_prefix, m_selectDirsDbRset.columnString("PATH"));
dir.set_name(dirname.basename);
dir.set_path(dirname.pathname);
......@@ -326,19 +324,6 @@ std::vector<eos::rpc::ContainerMdProto> EosImportDirs::getNextBatch() {
return dirs;
}
EosImportDirs::Dirname EosImportDirs::manglePathname(const std::string &pathname)
{
Dirname dir;
size_t clip = (pathname.find(m_castor_prefix) == 0) ? m_castor_prefix.length() : 0;
dir.pathname = m_eos_prefix + pathname.substr(clip);
clip = dir.pathname.find_last_of('/');
dir.basename = (clip == std::string::npos) ? dir.pathname : dir.pathname.substr(clip+1);
return dir;
}
}} // namespace cta::migration
......
......@@ -24,6 +24,7 @@
#include <XrdSsiPbLog.hpp>
#include "OracleDbConn.hpp"
#include "GrpcClient.hpp"
#include "GrpcUtils.hpp"
......@@ -97,8 +98,11 @@ EosImportFiles::EosImportFiles(const std::string &configfile) :
m_is_json = is_json.first ? is_json.second : false;
m_is_dry_run = is_dry_run.first ? is_dry_run.second : false;
m_batch_size = batch_size.first ? batch_size.second : 10000;
m_castor_prefix = castor_prefix.first ? castor_prefix.second : "/castor/cern.ch";
m_eos_prefix = eos_prefix.first ? eos_prefix.second : "/eos/grpc";
m_castor_prefix = castor_prefix.first ? castor_prefix.second : "/castor/cern.ch/";
m_eos_prefix = eos_prefix.first ? eos_prefix.second : "/eos/grpc/";
// enforce a slash at beginning and end of prefixes
eos::client::checkPrefix(m_castor_prefix);
eos::client::checkPrefix(m_eos_prefix);
}
......@@ -196,8 +200,9 @@ bool EosImportFiles::getNextBatch(std::vector<eos::rpc::FileMdProto> &files)
// BTIME is set as an extended attribute (see below)
// Filename and path
file.set_name(m_castordb.getResultColumnString("FILENAME"));
file.set_path(m_eos_prefix + m_castordb.getResultColumnString("PATH") + "/" + file.name());
auto p = eos::client::manglePathname(m_castor_prefix, m_eos_prefix, m_castordb.getResultColumnString("PATH"), m_castordb.getResultColumnString("FILENAME"));
file.set_name(p.basename);
file.set_path(p.pathname);
// we don't care about link_name
// Extended attributes:
......
......@@ -25,6 +25,7 @@
#include <XrdSsiPbConfig.hpp>
#include <XrdSsiPbLog.hpp>
#include "GrpcClient.hpp"
#include "GrpcUtils.hpp"
......@@ -42,14 +43,7 @@ public:
return m_eosgrpc->ping(payload);
}
struct Dirname {
std::string basename; //!< Just the directory name
std::string pathname; //!< The full path including the directory name
};
private:
Dirname manglePathname(const std::string &pathname);
std::chrono::steady_clock::time_point m_start_time; //!< Start the clock
bool m_is_json; //!< Display results in JSON format for debugging
......@@ -78,8 +72,11 @@ EosTestDirInject::EosTestDirInject(const std::string &configfile) :
// Set parameters and defaults
m_is_json = is_json.first ? is_json.second : false;
m_castor_prefix = castor_prefix.first ? castor_prefix.second : "/castor/cern.ch";
m_eos_prefix = eos_prefix.first ? eos_prefix.second : "/eos/grpc";
m_castor_prefix = castor_prefix.first ? castor_prefix.second : "/castor/cern.ch/";
m_eos_prefix = eos_prefix.first ? eos_prefix.second : "/eos/grpc/";
// enforce a slash at beginning and end of prefixes
eos::client::checkPrefix(m_castor_prefix);
eos::client::checkPrefix(m_eos_prefix);
}
......@@ -101,7 +98,7 @@ void EosTestDirInject::inject(const std::string &path, uint64_t fileid)
// we don't care about dir.stime (sync time, used for CERNBox)
// Directory name and full path
auto dirname = manglePathname(path);
auto dirname = eos::client::manglePathname(m_castor_prefix, m_eos_prefix, path);
dir.set_name(dirname.basename);
dir.set_path(dirname.pathname);
......@@ -131,19 +128,6 @@ void EosTestDirInject::inject(const std::string &path, uint64_t fileid)
std::cerr << "Processed 1 directory in " << elapsed_time.count() << "s" << std::endl;
}
EosTestDirInject::Dirname EosTestDirInject::manglePathname(const std::string &pathname)
{
Dirname dir;
size_t clip = (pathname.find(m_castor_prefix) == 0) ? m_castor_prefix.length() : 0;
dir.pathname = m_eos_prefix + pathname.substr(clip);
clip = dir.pathname.find_last_of('/');
dir.basename = (clip == std::string::npos) ? dir.pathname : dir.pathname.substr(clip+1);
return dir;
}
}} // namespace cta::migration
......
......@@ -22,6 +22,7 @@
#include <XrdSsiPbConfig.hpp>
#include <XrdSsiPbLog.hpp>
#include "GrpcClient.hpp"
#include "GrpcUtils.hpp"
......@@ -39,14 +40,8 @@ public:
return m_eosgrpc->ping(payload);
}
struct Dirname {
std::string basename; //!< Just the directory name
std::string pathname; //!< The full path including the directory name
};
private:
static std::string convertChecksum(uint32_t adler32);
Dirname manglePathname(const std::string &pathname);
std::chrono::steady_clock::time_point m_start_time; //!< Start the clock
......@@ -76,8 +71,11 @@ EosTestFileInject::EosTestFileInject(const std::string &configfile) :
// Set parameters and defaults
m_is_json = is_json.first ? is_json.second : false;
m_castor_prefix = castor_prefix.first ? castor_prefix.second : "/castor/cern.ch";
m_eos_prefix = eos_prefix.first ? eos_prefix.second : "/eos/grpc";
m_castor_prefix = castor_prefix.first ? castor_prefix.second : "/castor/cern.ch/";
m_eos_prefix = eos_prefix.first ? eos_prefix.second : "/eos/grpc/";
// enforce a slash at beginning and end of prefixes
eos::client::checkPrefix(m_castor_prefix);
eos::client::checkPrefix(m_eos_prefix);
}
......@@ -113,8 +111,9 @@ void EosTestFileInject::inject(const std::string &path, uint64_t fileid)
// BTIME is set as an extended attribute (see below)
// Filename and path
file.set_name(path);
file.set_path(m_eos_prefix + "/" + file.name());
auto p = eos::client::manglePathname(m_castor_prefix, m_eos_prefix, path);
file.set_name(p.basename);
file.set_path(p.pathname);
// we don't care about link_name
// Extended attributes:
......@@ -164,19 +163,6 @@ std::string EosTestFileInject::convertChecksum(uint32_t adler32)
return std::string(bytes, 4);
}
EosTestFileInject::Dirname EosTestFileInject::manglePathname(const std::string &pathname)
{
Dirname dir;
size_t clip = (pathname.find(m_castor_prefix) == 0) ? m_castor_prefix.length() : 0;
dir.pathname = m_eos_prefix + pathname.substr(clip);
clip = dir.pathname.find_last_of('/');
dir.basename = (clip == std::string::npos) ? dir.pathname : dir.pathname.substr(clip+1);
return dir;
}
}} // namespace cta::migration
......
/*!
* @project The CERN Tape Archive (CTA)
* @brief Utility class for gRPC injection tools
* @copyright Copyright 2019 CERN
* @license This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <string>
#include "GrpcUtils.hpp"
namespace eos {
namespace client {
void checkPrefix(std::string &prefix)
{
if(prefix.empty()) {
prefix = '/';
} else {
if(prefix.at(0) != '/') prefix = '/' + prefix;
if(prefix.at(prefix.length()-1) != '/') prefix += '/';
}
}
Dirname manglePathname(const std::string &remove_prefix, const std::string &add_prefix, const std::string &pathname, const std::string &filename)
{
Dirname dir;
// Set the pathname
size_t clip = (pathname.rfind(remove_prefix, 0) == std::string::npos) ? 0 : remove_prefix.length();
if(pathname.length() > clip && pathname.at(clip) == '/') ++clip;
dir.pathname = add_prefix + pathname.substr(clip);
// Set the filename
if(filename.empty()) {
clip = dir.pathname.find_last_of('/');
dir.basename = dir.pathname.substr(clip+1);
} else {
if(!dir.pathname.empty() && dir.pathname.at(pathname.length()-1) != '/') dir.pathname += '/';
dir.pathname += filename;
dir.basename = filename;
}
return dir;
}
}} // namespace eos::client
/*!
* @project The CERN Tape Archive (CTA)
* @brief Utility functions for gRPC injection tools
* @copyright Copyright 2019 CERN
* @license This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
namespace eos {
namespace client {
/*!
* Basename and pathname of a path
*/
struct Dirname {
std::string basename; //!< Just the directory name
std::string pathname; //!< The full path including the directory name
};
/*!
* Enforce prefixes to begin and end with a slash
*/
void checkPrefix(std::string &prefix);
/*!
* Remove the first prefix from pathname, prepend the second prefix, then split path into basename and pathname parts
*
* Note: prefixes must begin and end with a slash
*/
Dirname manglePathname(const std::string &remove_prefix, const std::string &add_prefix, const std::string &pathname, const std::string &filename = "");
}} // namespace eos::client
#include <iostream>
#include "GrpcUtils.hpp"
void test(const std::string &p1, const std::string &p2, const std::string &path, const std::string &filename = "")
{
auto d = eos::client::manglePathname(p1, p2, path, filename);
std::cout << "(" << p1 << ", " << p2 << ") " << path << " -> " << d.pathname << " (" << d.basename << ")" << std::endl;
}
int main()
{
std::string p1("");
std::string p3("/");
std::string p4("hello");
std::string p5("/hello");
std::string p6("hello/");
std::string p7("/hello/");
std::string p8("/hello/world/");
eos::client::checkPrefix(p1);
eos::client::checkPrefix(p3);
eos::client::checkPrefix(p4);
eos::client::checkPrefix(p5);
eos::client::checkPrefix(p6);
eos::client::checkPrefix(p7);
eos::client::checkPrefix(p8);
std::cout << p1 << std::endl;
std::cout << p3 << std::endl;
std::cout << p4 << std::endl;
std::cout << p5 << std::endl;
std::cout << p6 << std::endl;
std::cout << p7 << std::endl;
std::cout << p8 << std::endl;
test("/", "/", "");
test("/pre/", "/", "");
test("/", "/pre/", "");
test("/", "/", "/just/the/path");
test("/prefix/", "/", "/prefix/just/the/path");
test("/", "/just/", "/the/path");
test("/one/", "/two/", "/one/rest/of/the/path");
test("/substr/", "/two/", "/substring/rest/of/the/path");
test("/prefix/is/the/path/", "/", "/prefix/is/the/path/");
test("/prefix/is/the/path/", "/", "/prefix/is/the/path");
test("/prefix/is/longer/than/the/path/", "/newprefix/", "/the/path");
test("/", "/", "/path/with/no/filename/");
test("/", "/", "/path/and/a/filename/", "filename");
test("/", "/", "/path/and/a/filename", "filename");
}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment