Commit 24c16544 authored by Eric Cano's avatar Eric Cano
Browse files

Removed remotens directory, which was deprecated.

parent d8f41e5a
......@@ -119,7 +119,6 @@ ELSE(DEFINED PackageOnly)
add_subdirectory(mediachanger)
add_subdirectory(objectstore)
add_subdirectory(rdbms)
add_subdirectory(remotens)
add_subdirectory(scheduler)
add_subdirectory(tapeserver)
add_subdirectory(tests)
......
cmake_minimum_required (VERSION 2.6)
set (CTA_REMOTENS_SRC_FILES
EosNS.cpp
MockRemoteNS.cpp
RemoteNS.cpp
RemoteNSDispatcher.cpp)
add_library (ctaremotens SHARED
${CTA_REMOTENS_SRC_FILES})
find_package (xrootd REQUIRED)
include_directories (${XROOTD_INCLUDE_DIR} ${XROOTD_PRIVATE_INCLUDE_DIR} ${CMAKE_SOURCE_DIR})
target_link_libraries (ctaremotens ${XROOTD_XRDCL_LIB} ctacommon cryptopp ctautils)
install (TARGETS ctaremotens DESTINATION usr/${CMAKE_INSTALL_LIBDIR})
add_library (ctaremotensunittests SHARED
MockRemoteNS.cpp
MockRemoteFullFS.cpp
MockRemoteNSTest.cpp
RemoteNSDispatcherTest.cpp
RemoteNSFactory.cpp)
target_link_libraries(ctaremotensunittests
ctaremotens
ctautils)
install(TARGETS ctaremotensunittests DESTINATION usr/${CMAKE_INSTALL_LIBDIR})
/*
* The CERN Tape Archive (CTA) project
* Copyright (C) 2015 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <iostream>
#include <map>
#include <stdio.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include "common/exception/Exception.hpp"
#include "common/remoteFS/RemotePath.hpp"
#include "common/utils/utils.hpp"
#include "remotens/EosNS.hpp"
#include "XrdCl/XrdClFileSystem.hh"
#include "XrdCl/XrdClFile.hh"
#include <cryptopp/base64.h>
#include <cryptopp/osrng.h>
//------------------------------------------------------------------------------
// constructor
//------------------------------------------------------------------------------
cta::EosNS::EosNS(const std::string &XrootServerURL): m_xrootServerURL(XrootServerURL) {
}
//------------------------------------------------------------------------------
// destructor
//------------------------------------------------------------------------------
cta::EosNS::~EosNS() throw() {
}
//------------------------------------------------------------------------------
// statFile
//------------------------------------------------------------------------------
std::unique_ptr<cta::RemoteFileStatus> cta::EosNS::statFile(const RemotePath &path) const {
XrdCl::FileSystem fs{XrdCl::URL(m_xrootServerURL)};
XrdCl::StatInfo *stat_res = NULL;
XrdCl::XRootDStatus s = fs.Stat(path.getAfterScheme(), stat_res, 0);
if(!s.IsOK()) {
delete stat_res;
return std::unique_ptr<RemoteFileStatus>();
}
cta::common::dataStructures::UserIdentity owner("user0", "group0");
//here we cannot get the mode bits from EOS with stat_res->GetFlags() (xroot does not allow this to work properly)
//therefore we hardcode a fully permissive mode for the prototype
cta::RemoteFileStatus *rfs = new cta::RemoteFileStatus(owner, 0666, stat_res->GetSize());
delete stat_res;
return std::unique_ptr<RemoteFileStatus>(rfs);
}
//------------------------------------------------------------------------------
// rename
//------------------------------------------------------------------------------
void cta::EosNS::rename(const RemotePath &remoteFile, const RemotePath &newRemoteFile) {
XrdCl::FileSystem fs{XrdCl::URL(m_xrootServerURL)};
XrdCl::XRootDStatus s = fs.Mv(remoteFile.getAfterScheme(), newRemoteFile.getAfterScheme());
if(!s.IsOK()) {
throw cta::exception::Exception(std::string("Error renaming ")+remoteFile.getAfterScheme()+" to "+newRemoteFile.getAfterScheme()+" Reason: "+s.GetErrorMessage());
}
}
//------------------------------------------------------------------------------
// getFilename
//------------------------------------------------------------------------------
std::string cta::EosNS::getFilename(const RemotePath &remoteFile) const {
const std::string afterScheme = remoteFile.getAfterScheme();
const std::string afterSchemeTrimmed = utils::trimSlashes(afterScheme);
return utils::getEnclosedName(afterSchemeTrimmed);
}
//------------------------------------------------------------------------------
// createEntry
//------------------------------------------------------------------------------
void cta::EosNS::createEntry(const RemotePath &path, const RemoteFileStatus &status) {
XrdCl::File newFile;
XrdCl::Access::Mode mode;
if(status.mode & S_IRUSR) mode|= XrdCl::Access::Mode::UR;
if(status.mode & S_IWUSR) mode|= XrdCl::Access::Mode::UW;
if(status.mode & S_IXUSR) mode|= XrdCl::Access::Mode::UX;
if(status.mode & S_IRGRP) mode|= XrdCl::Access::Mode::GR;
if(status.mode & S_IWGRP) mode|= XrdCl::Access::Mode::GW;
if(status.mode & S_IXGRP) mode|= XrdCl::Access::Mode::GX;
if(status.mode & S_IROTH) mode|= XrdCl::Access::Mode::OR;
if(status.mode & S_IWOTH) mode|= XrdCl::Access::Mode::OW;
if(status.mode & S_IXOTH) mode|= XrdCl::Access::Mode::OX;
XrdCl::OpenFlags::Flags flags;
flags |= XrdCl::OpenFlags::Flags::Delete;
std::string eosPath(std::string("root://")+m_xrootServerURL+path.getAfterScheme());
XrdCl::XRootDStatus s = newFile.Open(eosPath, flags, mode);
if(!s.IsOK()) {
throw cta::exception::Exception(std::string("Error creating EOS file/dir ")+eosPath+" Reason: "+s.GetErrorMessage());
}
s = newFile.Close();
if(!s.IsOK()) {
throw cta::exception::Exception(std::string("Error closing EOS file/dir ")+eosPath+" Reason: "+s.GetErrorMessage());
}
}
/*
* The CERN Tape Archive (CTA) project
* Copyright (C) 2015 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "common/remoteFS/RemoteFileStatus.hpp"
#include "remotens/RemoteNS.hpp"
#include <map>
namespace cta {
/**
* A mock proxy class for the namespace of a remote storage system.
*/
class EosNS: public RemoteNS {
public:
/**
* Destructor.
*/
~EosNS() throw();
/**
* Constructor.
*/
EosNS(const std::string &XrootServerURL = "localhost:1094");
/**
* Returns the status of the specified file or directory within the remote
* storage system or NULL if the fil eor directory does not exist.
*
* @param path The absolute path of the file or directory.
* @return The status of the file or directory or NULL if the file or
* directory does not exist.
*/
std::unique_ptr<RemoteFileStatus> statFile(const RemotePath &path) const;
/**
* Renames the specified remote file to the specified new name.
*
* @param remoteFile The current path.
* @param newRemoteFile The new path.
*/
void rename(const RemotePath &remoteFile, const RemotePath &newRemoteFile);
/**
* Parses the specified remote file and returns the file name.
*
* @param path The absolute path of the file.
* @return The file name.
*/
std::string getFilename(const RemotePath &remoteFile) const;
/**
* Adds a directory or file to the remote NS with the desired status
*
* @param status The desired status of the entry to be created
* @param path The absolute path of the file or directory
*/
void createEntry(const RemotePath &path, const RemoteFileStatus &status);
private:
/**
* The xroot server URL
*/
std::string m_xrootServerURL;
}; // class EosNS
} // namespace cta
/*
* The CERN Tape Archive (CTA) project
* Copyright (C) 2015 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "MockRemoteFullFS.hpp"
#include "common/exception/Errnum.hpp"
#include <fcntl.h>
#include <unistd.h>
namespace cta {
MockRemoteFullFS::MockRemoteFullFS() {
char path[] = "/tmp/MockRemoteFullFS_XXXXXX";
exception::Errnum::throwOnNull(mkdtemp(path), "In MockRemoteFullFS::MockRemoteFullFS(): failed to create the temporary directory");
m_basePath = path;
std::string regex = std::string("file://") + path + "(/.+)";
m_pathRegex.reset(new castor::tape::utils::Regex(regex.c_str()));
}
std::string MockRemoteFullFS::createFullURL(const std::string& relativePath) {
if (relativePath[0] == '/')
return std::string ("file://") + m_basePath + relativePath;
else
return std::string ("file://") + m_basePath + "/" + relativePath;
}
void MockRemoteFullFS::createFile(const std::string& URL, size_t size) {
auto match = m_pathRegex->exec(URL);
if (match.empty())
throw exception::Exception("In MockRemoteFullFS::createFile(): invalid path");
std::string path = m_basePath + "/" + match.at(1);
// Create the file
int randomFd = ::open("/dev/urandom", 0);
exception::Errnum::throwOnMinusOne(randomFd, "Error opening /dev/urandom");
int fd = ::open(path.c_str(), O_CREAT | O_EXCL | O_WRONLY);
exception::Errnum::throwOnMinusOne(fd, "Error creating a file");
char * buff = NULL;
try {
buff = new char[size];
if(!buff) { throw exception::Exception("Failed to allocate memory to read /dev/urandom"); }
exception::Errnum::throwOnMinusOne(read(randomFd, buff, size));
exception::Errnum::throwOnMinusOne(write(fd, buff, size));
//m_checksum = adler32(0L, Z_NULL, 0);
//m_checksum = adler32(m_checksum, (const Bytef *)buff, size);
close(randomFd);
close(fd);
exception::Errnum::throwOnNonZero(::chmod(path.c_str(), 0777));
delete[] buff;
} catch (...) {
delete[] buff;
unlink(path.c_str());
throw;
}
}
void MockRemoteFullFS::deleteFile(const std::string& URL) {
auto match = m_pathRegex->exec(URL);
if (match.empty())
throw exception::Exception("In MockRemoteFullFS::deleteFile(): invalid path");
std::string path = m_basePath + "/" + match.at(1);
// delete the file
unlink(path.c_str());
}
std::string MockRemoteFullFS::getFilename(const RemotePath& remoteFile) const {
auto match = m_pathRegex->exec(remoteFile.getRaw());
if (match.size())
return match.at(1);
throw exception::Exception("In MockRemoteFullFS::getFilename(): invalid path");
}
void MockRemoteFullFS::rename(const RemotePath& remoteFile, const RemotePath& newRemoteFile) {
auto srcMatch = m_pathRegex->exec(remoteFile.getRaw());
auto dstMatch = m_pathRegex->exec(newRemoteFile.getRaw());
if (srcMatch.empty() || dstMatch.empty())
throw exception::Exception("In MockRemoteFullFS::rename(): invalid path");
std::string srcPath = m_basePath + srcMatch.at(1);
std::string dstPath = m_basePath + dstMatch.at(1);
exception::Errnum::throwOnNonZero(::rename(srcPath.c_str(), dstPath.c_str()));
}
std::unique_ptr<RemoteFileStatus> MockRemoteFullFS::statFile(const RemotePath& path) const {
std::unique_ptr<RemoteFileStatus> ret (new RemoteFileStatus);
auto pathMatch = m_pathRegex->exec(path.getRaw());
if (pathMatch.empty())
throw exception::Exception("In MockRemoteFullFS::statFile(): invalid path");
std::string realPath = m_basePath + pathMatch.at(1);
struct ::stat statBuff;
exception::Errnum::throwOnNonZero(::stat(realPath.c_str(), &statBuff));
ret->mode = statBuff.st_mode;
std::stringstream itoa;
itoa << statBuff.st_uid;
ret->owner.name = itoa.str();
itoa.clear();
itoa << statBuff.st_gid;
ret->owner.group = itoa.str();
ret->size = statBuff.st_size;
return ret;
}
int MockRemoteFullFS::deleteFileOrDirectory(const char* fpath,
const struct ::stat* sb, int tflag, struct ::FTW * ftwbuf) {
switch (tflag) {
case FTW_D:
case FTW_DNR:
case FTW_DP:
rmdir(fpath);
break;
default:
unlink(fpath);
break;
}
return 0;
}
MockRemoteFullFS::~MockRemoteFullFS() throw() {
// Delete the created directories recursively
nftw(m_basePath.c_str(), deleteFileOrDirectory, 100, FTW_DEPTH);
}
} // end of namespace cta
/*
* The CERN Tape Archive (CTA) project
* Copyright (C) 2015 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "common/remoteFS/RemoteFileStatus.hpp"
#include "remotens/RemoteNS.hpp"
#include "tapeserver/castor/tape/tapeserver/utils/Regex.hpp"
#include <sys/stat.h>
#include <ftw.h>
namespace cta {
/**
* A mock proxy class for the namespace of a remote storage system, backed
* by a real local temporary file system. This mock namespace also generates
* valid URLs in order for the tape server to be able to actually transfer
* data.
*/
class MockRemoteFullFS: public RemoteNS {
public:
/**
* Destructor.
*/
~MockRemoteFullFS() throw();
/**
* Constructor.
*/
MockRemoteFullFS();
/**
* Returns the status of the specified file or directory within the remote
* storage system or NULL if the fil eor directory does not exist.
*
* @param path The absolute path of the file or directory.
* @return The status of the file or directory or NULL if the file or
* directory does not exist.
*/
std::unique_ptr<RemoteFileStatus> statFile(const RemotePath &path) const;
/**
* Renames the specified remote file to the specified new name.
*
* @param remoteFile The current path.
* @param newRemoteFile The new path.
*/
void rename(const RemotePath &remoteFile,
const RemotePath &newRemoteFile);
/**
* Parses the specified remote file and returns the file name.
*
* @param path The absolute path of the file.
* @return The file name.
*/
std::string getFilename(const RemotePath &remoteFile) const;
/**
* Get a real URL that will be compatible with both remoteNS interface for
* metadata access and tape server's file access (via a file:// URL).
*/
std::string createFullURL(const std::string & relativePath);
/**
* Create a file with random content
*/
void createFile(const std::string & URL, size_t size);
/**
* Delete a file
*/
void deleteFile(const std::string & URL);
private:
std::string m_basePath; ///< The path to the directory within which all the URLs will be created
std::unique_ptr<castor::tape::utils::Regex> m_pathRegex; ///< A regex alloging validation of the path in calls
/** Utility function to delete a directory in depth */
static int deleteFileOrDirectory(const char* fpath,
const struct ::stat* sb, int tflag, struct ::FTW * ftwbuf);
}; // class MockRemoteNS
} // namespace cta
/*
* The CERN Tape Archive (CTA) project
* Copyright (C) 2015 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <map>
#include <stdio.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include "common/exception/Exception.hpp"
#include "common/remoteFS/RemotePath.hpp"
#include "common/utils/utils.hpp"
#include "remotens/MockRemoteNS.hpp"
//------------------------------------------------------------------------------
// constructor
//------------------------------------------------------------------------------
cta::MockRemoteNS::MockRemoteNS() {
}
//------------------------------------------------------------------------------
// constructor
//------------------------------------------------------------------------------
cta::MockRemoteNS::MockRemoteNS(std::map<RemotePath, RemoteFileStatus> &entries): m_entries(entries){
}
//------------------------------------------------------------------------------
// destructor
//------------------------------------------------------------------------------
cta::MockRemoteNS::~MockRemoteNS() throw() {
}
//------------------------------------------------------------------------------
// statFile
//------------------------------------------------------------------------------
std::unique_ptr<cta::RemoteFileStatus> cta::MockRemoteNS::statFile(
const RemotePath &path) const {
auto it = m_entries.find(path);
if(m_entries.end() == it) {
return std::unique_ptr<RemoteFileStatus>();
}
return std::unique_ptr<RemoteFileStatus>(
new RemoteFileStatus(m_entries.at(path)));
}
//------------------------------------------------------------------------------
// rename
//------------------------------------------------------------------------------
void cta::MockRemoteNS::rename(const RemotePath &remoteFile,
const RemotePath &newRemoteFile) {
auto it = m_entries.find(newRemoteFile);
if(m_entries.end() != it) {
throw exception::Exception("MockRemoteNS: destination path exists, cannot overwrite it");
}
it = m_entries.find(remoteFile);
if(m_entries.end() == it) {
throw exception::Exception("MockRemoteNS: source path does not exist");
}
m_entries[newRemoteFile] = m_entries[remoteFile];
m_entries.erase(remoteFile);
}
//------------------------------------------------------------------------------
// getFilename
//------------------------------------------------------------------------------
std::string cta::MockRemoteNS::getFilename(const RemotePath &remoteFile) const {
const std::string afterScheme = remoteFile.getAfterScheme();
const std::string afterSchemeTrimmed = utils::trimSlashes(afterScheme);
return utils::getEnclosedName(afterSchemeTrimmed);
}
//------------------------------------------------------------------------------
// createEntry
//------------------------------------------------------------------------------
void cta::MockRemoteNS::createEntry(const RemotePath &path,
const RemoteFileStatus &status){
auto it = m_entries.find(path);
if(m_entries.end() != it) {
throw exception::Exception("MockRemoteNS: path exists, cannot overwrite it");
}
m_entries[path] = status;
}
/*
* The CERN Tape Archive (CTA) project
* Copyright (C) 2015 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "common/remoteFS/RemoteFileStatus.hpp"
#include "remotens/RemoteNS.hpp"
#include <map>
namespace cta {
/**
* A mock proxy class for the namespace of a remote storage system.
*/
class MockRemoteNS: public RemoteNS {
public: