Commit 895e1586 authored by Michael Davis's avatar Michael Davis
Browse files

Merge remote-tracking branch 'origin/master' into ssi_ar_ls

parents 85ba58a6 ee211b54
0:eos-archive-4.1.26-20170824172038gitc7cf4c6.el7.cern.x86_64
0:eos-cleanup-4.1.26-20170824172038gitc7cf4c6.el7.cern.x86_64
0:eos-client-4.1.26-20170824172038gitc7cf4c6.el7.cern.x86_64
0:eos-debuginfo-4.1.26-20170824172038gitc7cf4c6.el7.cern.x86_64
0:eos-fuse-4.1.26-20170824172038gitc7cf4c6.el7.cern.x86_64
0:eos-fuse-core-4.1.26-20170824172038gitc7cf4c6.el7.cern.x86_64
0:eos-fuse-sysv-4.1.26-20170824172038gitc7cf4c6.el7.cern.x86_64
0:eos-server-4.1.26-20170824172038gitc7cf4c6.el7.cern.x86_64
0:eos-srm-4.1.26-20170824172038gitc7cf4c6.el7.cern.x86_64
0:eos-test-4.1.26-20170824172038gitc7cf4c6.el7.cern.x86_64
0:eos-testkeytab-4.1.26-20170824172038gitc7cf4c6.el7.cern.x86_64
0:eos-archive-4.2.4-20171206151625gitda6f67d.el7.cern.x86_64
0:eos-cleanup-4.2.4-20171206151625gitda6f67d.el7.cern.x86_64
0:eos-client-4.2.4-20171206151625gitda6f67d.el7.cern.x86_64
0:eos-debuginfo-4.2.4-20171206151625gitda6f67d.el7.cern.x86_64
0:eos-fuse-4.2.4-20171206151625gitda6f67d.el7.cern.x86_64
0:eos-fuse-core-4.2.4-20171206151625gitda6f67d.el7.cern.x86_64
0:eos-fuse-sysv-4.2.4-20171206151625gitda6f67d.el7.cern.x86_64
0:eos-fusex-4.2.4-20171206151625gitda6f67d.el7.cern.x86_64
0:eos-fusex-core-4.2.4-20171206151625gitda6f67d.el7.cern.x86_64
0:eos-fusex-selinux-4.2.4-20171206151625gitda6f67d.el7.cern.x86_64
0:eos-server-4.2.4-20171206151625gitda6f67d.el7.cern.x86_64
0:eos-srm-4.2.4-20171206151625gitda6f67d.el7.cern.x86_64
0:eos-test-4.2.4-20171206151625gitda6f67d.el7.cern.x86_64
0:eos-testkeytab-4.2.4-20171206151625gitda6f67d.el7.cern.x86_64
1:python2-xrootd-4.8.0-0.rc1.el7.x86_64
1:python3-xrootd-4.8.0-0.rc1.el7.x86_64
1:xrootd-4.8.0-0.rc1.el7.x86_64
......@@ -26,43 +29,43 @@
1:xrootd-server-devel-4.8.0-0.rc1.el7.x86_64
1:xrootd-server-libs-4.8.0-0.rc1.el7.x86_64
1:xrootd-tests-4.8.0-0.rc1.el7.x86_64
2:ceph-12.2.0-0.el7.x86_64
2:ceph-base-12.2.0-0.el7.x86_64
2:ceph-common-12.2.0-0.el7.x86_64
2:ceph-debuginfo-12.2.0-0.el7.x86_64
2:cephfs-java-12.2.0-0.el7.x86_64
2:ceph-fuse-12.2.0-0.el7.x86_64
2:ceph-mds-12.2.0-0.el7.x86_64
2:ceph-mgr-12.2.0-0.el7.x86_64
2:ceph-mon-12.2.0-0.el7.x86_64
2:ceph-osd-12.2.0-0.el7.x86_64
2:ceph-radosgw-12.2.0-0.el7.x86_64
2:ceph-resource-agents-12.2.0-0.el7.x86_64
2:ceph-selinux-12.2.0-0.el7.x86_64
2:ceph-test-12.2.0-0.el7.x86_64
2:libcephfs2-12.2.0-0.el7.x86_64
2:libcephfs-devel-12.2.0-0.el7.x86_64
2:libcephfs_jni1-12.2.0-0.el7.x86_64
2:libcephfs_jni-devel-12.2.0-0.el7.x86_64
2:librados2-12.2.0-0.el7.x86_64
2:librados-devel-12.2.0-0.el7.x86_64
2:libradosstriper1-12.2.0-0.el7.x86_64
2:libradosstriper-devel-12.2.0-0.el7.x86_64
2:librbd1-12.2.0-0.el7.x86_64
2:librbd-devel-12.2.0-0.el7.x86_64
2:librgw2-12.2.0-0.el7.x86_64
2:librgw-devel-12.2.0-0.el7.x86_64
2:python34-ceph-argparse-12.2.0-0.el7.x86_64
2:python34-cephfs-12.2.0-0.el7.x86_64
2:python34-rados-12.2.0-0.el7.x86_64
2:python34-rbd-12.2.0-0.el7.x86_64
2:python34-rgw-12.2.0-0.el7.x86_64
2:python-ceph-compat-12.2.0-0.el7.x86_64
2:python-cephfs-12.2.0-0.el7.x86_64
2:python-rados-12.2.0-0.el7.x86_64
2:python-rbd-12.2.0-0.el7.x86_64
2:python-rgw-12.2.0-0.el7.x86_64
2:rados-objclass-devel-12.2.0-0.el7.x86_64
2:rbd-fuse-12.2.0-0.el7.x86_64
2:rbd-mirror-12.2.0-0.el7.x86_64
2:rbd-nbd-12.2.0-0.el7.x86_64
2:ceph-12.2.2-0.el7.x86_64
2:ceph-base-12.2.2-0.el7.x86_64
2:ceph-common-12.2.2-0.el7.x86_64
2:ceph-debuginfo-12.2.2-0.el7.x86_64
2:cephfs-java-12.2.2-0.el7.x86_64
2:ceph-fuse-12.2.2-0.el7.x86_64
2:ceph-mds-12.2.2-0.el7.x86_64
2:ceph-mgr-12.2.2-0.el7.x86_64
2:ceph-mon-12.2.2-0.el7.x86_64
2:ceph-osd-12.2.2-0.el7.x86_64
2:ceph-radosgw-12.2.2-0.el7.x86_64
2:ceph-resource-agents-12.2.2-0.el7.x86_64
2:ceph-selinux-12.2.2-0.el7.x86_64
2:ceph-test-12.2.2-0.el7.x86_64
2:libcephfs2-12.2.2-0.el7.x86_64
2:libcephfs-devel-12.2.2-0.el7.x86_64
2:libcephfs_jni1-12.2.2-0.el7.x86_64
2:libcephfs_jni-devel-12.2.2-0.el7.x86_64
2:librados2-12.2.2-0.el7.x86_64
2:librados-devel-12.2.2-0.el7.x86_64
2:libradosstriper1-12.2.2-0.el7.x86_64
2:libradosstriper-devel-12.2.2-0.el7.x86_64
2:librbd1-12.2.2-0.el7.x86_64
2:librbd-devel-12.2.2-0.el7.x86_64
2:librgw2-12.2.2-0.el7.x86_64
2:librgw-devel-12.2.2-0.el7.x86_64
2:python34-ceph-argparse-12.2.2-0.el7.x86_64
2:python34-cephfs-12.2.2-0.el7.x86_64
2:python34-rados-12.2.2-0.el7.x86_64
2:python34-rbd-12.2.2-0.el7.x86_64
2:python34-rgw-12.2.2-0.el7.x86_64
2:python-ceph-compat-12.2.2-0.el7.x86_64
2:python-cephfs-12.2.2-0.el7.x86_64
2:python-rados-12.2.2-0.el7.x86_64
2:python-rbd-12.2.2-0.el7.x86_64
2:python-rgw-12.2.2-0.el7.x86_64
2:rados-objclass-devel-12.2.2-0.el7.x86_64
2:rbd-fuse-12.2.2-0.el7.x86_64
2:rbd-mirror-12.2.2-0.el7.x86_64
2:rbd-nbd-12.2.2-0.el7.x86_64
......@@ -158,7 +158,7 @@ test -e /usr/lib64/libjemalloc.so.1 && export LD_PRELOAD=/usr/lib64/libjemalloc.
# ACLs in EOS are evaluated when unix permissions are failing, hence the 555 unix permission.
eos mkdir ${CTA_TEST_DIR}
eos chmod 555 ${CTA_TEST_DIR}
eos attr set sys.acl=g:eosusers:rwx!d,g:powerusers:rwx+d /eos/ctaeos/cta
eos attr set sys.acl=g:eosusers:rwx!d,u:poweruser1:rwx+dp,u:poweruser2:rwx+dp /eos/ctaeos/cta
eos attr set CTA_StorageClass=ctaStorageClass ${CTA_TEST_DIR}
......
......@@ -8,7 +8,7 @@ chmod 755 /var/eos/wfe/bash/*
eos mkdir ${PREPROD_DIR}
eos chmod 555 ${PREPROD_DIR}
eos attr set sys.acl=g:eosusers:rwx!d,g:powerusers:rwx+d ${PREPROD_DIR}
eos attr set sys.acl=g:eosusers:rwx!d,u:poweruser1:rwx+dp,u:poweruser2:rwx+dp ${PREPROD_DIR}
eos attr set CTA_StorageClass=ctaStorageClass ${PREPROD_DIR}
......
......@@ -35,6 +35,7 @@ echo "Launching simple_client_ar.sh on client pod"
echo " Archiving file: xrdcp as user1"
echo " Retrieving it as poweruser1"
kubectl -n ${NAMESPACE} cp simple_client_ar.sh client:/root/client_ar.sh
kubectl -n ${NAMESPACE} cp client_helper.sh client:/root/client_helper.sh
kubectl -n ${NAMESPACE} exec client -- bash /root/simple_client_ar.sh || exit 1
NB_FILES=1000
......@@ -46,7 +47,6 @@ echo " Archiving ${NB_FILES} files of ${FILE_SIZE_KB}kB each"
echo " Archiving files: xrdcp as user1"
echo " Retrieving them as poweruser1"
kubectl -n ${NAMESPACE} cp client_ar.sh client:/root/client_ar.sh
kubectl -n ${NAMESPACE} cp client_helper.sh client:/root/client_helper.sh
kubectl -n ${NAMESPACE} exec client -- bash /root/client_ar.sh -n ${NB_FILES} -s ${FILE_SIZE_KB} -p 10 -d /eos/ctaeos/preprod -v -r || exit 1
exit 0
......@@ -76,6 +76,14 @@ if [[ $VERBOSE == 1 ]]; then
TAILPID=$!
fi
# get some common useful helpers for krb5
. /root/client_helper.sh
# Get kerberos credentials for poweruser1
eospower_kdestroy
eospower_kinit
EOS_DIR="${EOS_BASEDIR}/$(uuidgen)"
echo "Creating test dir in eos: ${EOS_DIR}"
# uuid should be unique no need to remove dir before...
......@@ -141,7 +149,8 @@ echo "Trigerring EOS retrieve workflow as poweruser1:powerusers (12001:1200)"
# XrdSecPROTOCOL=sss xrdfs ${EOSINSTANCE} prepare -s "${EOS_DIR}/${TEST_FILE_NAME}?eos.ruid=12001&eos.rgid=1200" || echo "Could not trigger retrieve for ${EOS_DIR}/${TEST_FILE_NAME}"
#done
grep tapeonly$ ${STATUS_FILE} | sed -e 's/ .*$//' | XrdSecPROTOCOL=sss xargs --max-procs=${NB_PROCS} -iTEST_FILE_NAME xrdfs ${EOSINSTANCE} prepare -s "${EOS_DIR}/TEST_FILE_NAME?eos.ruid=12001&eos.rgid=1200"
# We need the -s as we are staging the files from tape (see xrootd prepare definition)
grep tapeonly$ ${STATUS_FILE} | sed -e 's/ .*$//' | KRB5CCNAME=/tmp/${EOSPOWER_USER}/krb5cc_0 XrdSecPROTOCOL=krb5 xargs --max-procs=${NB_PROCS} -iTEST_FILE_NAME xrdfs ${EOSINSTANCE} prepare -s ${EOS_DIR}/TEST_FILE_NAME
# Wait for the copy to appear on disk
......
......@@ -4,6 +4,11 @@
EOSINSTANCE=ctaeos
TEST_FILE_NAME=`uuidgen`
# get some common useful helpers for krb5
. /root/client_helper.sh
eospower_kdestroy
eospower_kinit
echo "xrdcp /etc/group root://${EOSINSTANCE}//eos/ctaeos/cta/${TEST_FILE_NAME}"
xrdcp /etc/group root://${EOSINSTANCE}//eos/ctaeos/cta/${TEST_FILE_NAME}
......@@ -41,9 +46,11 @@ echo "Information about the testing file without disk replica"
eos root://${EOSINSTANCE} info /eos/ctaeos/cta/${TEST_FILE_NAME}
echo
echo "Trigerring EOS retrieve workflow as poweruser1:powerusers (12001:1200)"
echo "XrdSecPROTOCOL=sss xrdfs ${EOSINSTANCE} prepare -s \"/eos/ctaeos/cta/${TEST_FILE_NAME}?eos.ruid=12001&eos.rgid=1200\""
XrdSecPROTOCOL=sss xrdfs ${EOSINSTANCE} prepare -s "/eos/ctaeos/cta/${TEST_FILE_NAME}?eos.ruid=12001&eos.rgid=1200"
#echo "XrdSecPROTOCOL=sss xrdfs ${EOSINSTANCE} prepare -s \"/eos/ctaeos/cta/${TEST_FILE_NAME}?eos.ruid=12001&eos.rgid=1200\""
# XrdSecPROTOCOL=sss xrdfs ${EOSINSTANCE} prepare -s "/eos/ctaeos/cta/${TEST_FILE_NAME}?eos.ruid=12001&eos.rgid=1200"
# We need the -s as we are staging the files from tape (see xrootd prepare definition)
KRB5CCNAME=/tmp/${EOSPOWER_USER}/krb5cc_0 XrdSecPROTOCOL=krb5 xrdfs ${EOSINSTANCE} prepare -s /eos/ctaeos/cta/${TEST_FILE_NAME}
# Wait for the copy to appear on disk
SECONDS_PASSED=0
......
......@@ -23,7 +23,7 @@
#include "common/utils/utils.hpp"
#include "common/utils/Regex.hpp"
auto cta::objectstore::BackendFactory::createBackend(const std::string& URL)
auto cta::objectstore::BackendFactory::createBackend(const std::string& URL, log::Logger & logger)
-> std::unique_ptr<Backend> {
utils::Regex fileRe("^file://(.*)$"),
radosRe("^rados://([^@]+)@([^:]+)(|:(.+))$");
......@@ -39,9 +39,9 @@ auto cta::objectstore::BackendFactory::createBackend(const std::string& URL)
if (regexResult.size() != 5 && regexResult.size() != 4)
throw cta::exception::Exception("In BackendFactory::createBackend(): unexpected number of matches in regex");
if (regexResult.size() == 5)
return std::unique_ptr<Backend>(new BackendRados(regexResult[1], regexResult[2], regexResult[4]));
return std::unique_ptr<Backend>(new BackendRados(logger, regexResult[1], regexResult[2], regexResult[4]));
else
return std::unique_ptr<Backend>(new BackendRados(regexResult[1], regexResult[2]));
return std::unique_ptr<Backend>(new BackendRados(logger, regexResult[1], regexResult[2]));
}
// Fall back to a file URL if all failed
return std::unique_ptr<Backend>(new BackendVFS(URL));
......
......@@ -19,11 +19,12 @@
#pragma once
#include "Backend.hpp"
#include "common/log/Logger.hpp"
#include <memory>
namespace cta { namespace objectstore {
class BackendFactory {
public:
static std::unique_ptr<Backend> createBackend(const std::string & URL);
static std::unique_ptr<Backend> createBackend(const std::string & URL, log::Logger & logger);
};
}}
\ No newline at end of file
......@@ -80,23 +80,53 @@ namespace cta { namespace objectstore {
cta::threading::Mutex BackendRados::RadosTimeoutLogger::g_mutex;
BackendRados::BackendRados(const std::string & userId, const std::string & pool, const std::string &radosNameSpace) :
BackendRados::BackendRados(log::Logger & logger, const std::string & userId, const std::string & pool,
const std::string &radosNameSpace) :
m_user(userId), m_pool(pool), m_namespace(radosNameSpace), m_cluster(), m_radosCtx() {
cta::exception::Errnum::throwOnReturnedErrno(-m_cluster.init(userId.c_str()),
"In ObjectStoreRados::ObjectStoreRados, failed to m_cluster.init");
bool contextSet=false;
try {
RadosTimeoutLogger rtl;
cta::exception::Errnum::throwOnReturnedErrno(-m_cluster.conf_read_file(NULL),
"In ObjectStoreRados::ObjectStoreRados, failed to m_cluster.conf_read_file");
rtl.logIfNeeded("In BackendRados::BackendRados(): m_cluster.conf_read_file()", "no object");
rtl.reset();
cta::exception::Errnum::throwOnReturnedErrno(-m_cluster.conf_parse_env(NULL),
"In ObjectStoreRados::ObjectStoreRados, failed to m_cluster.conf_parse_env");
rtl.logIfNeeded("In BackendRados::BackendRados(): m_cluster.conf_parse_env()", "no object");
rtl.reset();
cta::exception::Errnum::throwOnReturnedErrno(-m_cluster.connect(),
"In ObjectStoreRados::ObjectStoreRados, failed to m_cluster.connect");
rtl.logIfNeeded("In BackendRados::BackendRados(): m_cluster.connect()", "no object");
rtl.reset();
cta::exception::Errnum::throwOnReturnedErrno(-m_cluster.ioctx_create(pool.c_str(), m_radosCtx),
"In ObjectStoreRados::ObjectStoreRados, failed to m_cluster.ioctx_create");
rtl.logIfNeeded("In BackendRados::BackendRados(): m_cluster.ioctx_create()", "no object");
contextSet=true;
// An empty string also sets the namespace to default so no need to filter. This function does not fail.
m_radosCtx.set_namespace(radosNameSpace);
// Create the thread pool. One thread per CPU hardware thread.
for (size_t i=0; i<std::thread::hardware_concurrency(); i++) {
RadosWorkerThreadAndContext * rwtac = new RadosWorkerThreadAndContext(m_cluster, pool, radosNameSpace, i, logger);
m_threads.push_back(rwtac);
m_threads.back()->start();
}
} catch (...) {
for (size_t i=0; i<m_threads.size(); i++) m_JobQueue.push(nullptr);
for (auto &t: m_threads) {
if (t) t->wait();
delete t;
}
if (contextSet)
{
RadosTimeoutLogger rtl;
m_radosCtx.close();
rtl.logIfNeeded("In BackendRados::BackendRados(): m_radosCtx.close()", "no object");
}
RadosTimeoutLogger rtl;
m_cluster.shutdown();
rtl.logIfNeeded("In BackendRados::BackendRados(): m_cluster.shutdown()", "no object");
throw;
}
}
......@@ -504,6 +534,28 @@ BackendRados::ScopedLock* BackendRados::lockShared(std::string name, uint64_t ti
return ret.release();
}
BackendRados::RadosWorkerThreadAndContext::RadosWorkerThreadAndContext(librados::Rados& cluster,
const std::string & pool, const std::string& radosNameSpace, int threadID, log::Logger & logger):
m_threadID(threadID), m_lc(logger) {
RadosTimeoutLogger rtl;
cta::exception::Errnum::throwOnReturnedErrno(-cluster.ioctx_create(pool.c_str(), m_radosCtx),
"In RadosWorkerThreadAndContext::RadosWorkerThreadAndContext, failed to cluster.ioctx_create");
rtl.logIfNeeded("In RadosWorkerThreadAndContext::RadosWorkerThreadAndContext(): ", "no object");
// An empty string also sets the namespace to default so no need to filter. This function does not fail.
m_radosCtx.set_namespace(radosNameSpace);
}
BackendRados::RadosWorkerThreadAndContext::~RadosWorkerThreadAndContext() {
RadosTimeoutLogger rtl;
m_radosCtx.close();
rtl.logIfNeeded("In RadosWorkerThreadAndContext::~RadosWorkerThreadAndContext(): m_radosCtx.close()", "no object");
}
void BackendRados::RadosWorkerThreadAndContext::run() {
}
Backend::AsyncUpdater* BackendRados::asyncUpdate(const std::string & name, std::function <std::string(const std::string &)> & update)
{
return new AsyncUpdater(*this, name, update);
......
......@@ -21,6 +21,9 @@
#include "Backend.hpp"
#include "rados/librados.hpp"
#include "common/threading/Mutex.hpp"
#include "common/threading/BlockingQueue.hpp"
#include "common/log/Logger.hpp"
#include "common/log/LogContext.hpp"
#include <future>
// RADOS_LOCKING can be NOTIFY or BACKOFF
......@@ -57,7 +60,7 @@ public:
* @param userId
* @param pool
*/
BackendRados(const std::string & userId, const std::string & pool, const std::string &radosNameSpace = "");
BackendRados(log::Logger & logger, const std::string & userId, const std::string & pool, const std::string &radosNameSpace = "");
~BackendRados() override;
std::string user() {
return m_user;
......@@ -183,6 +186,42 @@ private:
uint64_t m_watchHandle;
};
private:
/**
* Base class for jobs handled by the thread-and-context pool.
*/
class AsyncJob {
virtual void execute(librados::IoCtx & context)=0;
virtual ~AsyncJob() {}
};
/**
* The queue for the thread-and-context pool.
*/
cta::threading::BlockingQueue<AsyncJob *> m_JobQueue;
/**
* The class for the worker threads
*/
class RadosWorkerThreadAndContext: private cta::threading::Thread {
public:
RadosWorkerThreadAndContext(librados::Rados & cluster, const std::string & pool, const std::string & radosNameSpace,
int threadID, log::Logger & logger);
virtual ~RadosWorkerThreadAndContext();
void start() { cta::threading::Thread::start(); }
void wait() { cta::threading::Thread::wait(); }
private:
librados::IoCtx m_radosCtx;
const int m_threadID;
log::LogContext m_lc;
void run() override;
};
/**
* The container for the threads
*/
std::vector<RadosWorkerThreadAndContext *> m_threads;
public:
/**
* A class following up the check existence-lock-fetch-update-write-unlock. Constructor implicitly
......
......@@ -44,7 +44,7 @@ int main(int argc, char ** argv) {
cta::log::LogContext lc(sl);
std::string objectName;
if (4 == argc) {
be.reset(cta::objectstore::BackendFactory::createBackend(argv[1]).release());
be.reset(cta::objectstore::BackendFactory::createBackend(argv[1], sl).release());
const cta::rdbms::Login catalogueLogin = cta::rdbms::Login::parseFile(argv[2]);
const uint64_t nbConns = 1;
const uint64_t nbArchiveFileListingConns = 0;
......@@ -52,7 +52,7 @@ int main(int argc, char ** argv) {
objectName = argv[3];
} else if (2 == argc) {
cta::common::Configuration m_ctaConf("/etc/cta/cta-frontend.conf");
be = std::move(cta::objectstore::BackendFactory::createBackend(m_ctaConf.getConfEntString("ObjectStore", "BackendPath", nullptr)));
be = std::move(cta::objectstore::BackendFactory::createBackend(m_ctaConf.getConfEntString("ObjectStore", "BackendPath", nullptr), sl));
const cta::rdbms::Login catalogueLogin = cta::rdbms::Login::parseFile("/etc/cta/cta-catalogue.conf");
const uint64_t nbConns = 1;
const uint64_t nbArchiveFileListingConns = 0;
......
......@@ -23,6 +23,8 @@
*/
#include "common/Configuration.hpp"
#include "common/log/StdoutLogger.hpp"
#include "common/log/LogContext.hpp"
#include "BackendFactory.hpp"
#include "BackendVFS.hpp"
#include "Agent.hpp"
......@@ -33,12 +35,13 @@
int main(int argc, char ** argv) {
try {
cta::log::StdoutLogger logger("cta-objectstore-dereference-removed-queues");
std::unique_ptr<cta::objectstore::Backend> be;
if (2 == argc) {
be.reset(cta::objectstore::BackendFactory::createBackend(argv[1]).release());
be.reset(cta::objectstore::BackendFactory::createBackend(argv[1], logger).release());
} else if (1 == argc) {
cta::common::Configuration m_ctaConf("/etc/cta/cta-frontend.conf");
be = std::move(cta::objectstore::BackendFactory::createBackend(m_ctaConf.getConfEntString("ObjectStore", "BackendPath", nullptr)));
be = std::move(cta::objectstore::BackendFactory::createBackend(m_ctaConf.getConfEntString("ObjectStore", "BackendPath", nullptr), logger));
} else {
throw std::runtime_error("Wrong number of arguments: expected 0 or 1: [objectstoreURL]");
}
......
......@@ -24,6 +24,8 @@
#include "common/Configuration.hpp"
#include "BackendFactory.hpp"
#include "common/log/DummyLogger.hpp"
#include "common/log/LogContext.hpp"
#include "BackendVFS.hpp"
#include "GenericObject.hpp"
#include <iostream>
......@@ -31,14 +33,15 @@
int main(int argc, char ** argv) {
try {
cta::log::DummyLogger dl("");
std::unique_ptr<cta::objectstore::Backend> be;
std::string objectName;
if (3 == argc) {
be.reset(cta::objectstore::BackendFactory::createBackend(argv[1]).release());
be.reset(cta::objectstore::BackendFactory::createBackend(argv[1], dl).release());
objectName = argv[2];
} else if (2 == argc ){
cta::common::Configuration m_ctaConf("/etc/cta/cta-frontend.conf");
be=std::move(cta::objectstore::BackendFactory::createBackend(m_ctaConf.getConfEntString("ObjectStore", "BackendPath", nullptr)));
be=std::move(cta::objectstore::BackendFactory::createBackend(m_ctaConf.getConfEntString("ObjectStore", "BackendPath", nullptr), dl));
objectName = argv[1];
} else {
throw std::runtime_error("Wrong number of arguments: expected 1 or 2: [objectstoreURL] objectname");
......
......@@ -33,12 +33,13 @@
int main(int argc, char ** argv) {
try {
cta::log::StdoutLogger logger("cta-objectstore-initialize");
cta::log::LogContext lc(logger);
std::unique_ptr<cta::objectstore::Backend> be;
if (1 == argc) {
be.reset(new cta::objectstore::BackendVFS);
} else if (2 == argc) {
be.reset(cta::objectstore::BackendFactory::createBackend(argv[1]).release());
be.reset(cta::objectstore::BackendFactory::createBackend(argv[1], logger).release());
} else {
throw std::runtime_error("Wrong number of arguments: expected 0 or 1");
}
......@@ -52,12 +53,10 @@ int main(int argc, char ** argv) {
re.insert();
cta::objectstore::ScopedExclusiveLock rel(re);
re.fetch();
cta::log::StdoutLogger logger("cta-objectstore-initialize");
cta::objectstore::AgentReference agr("cta-objectstore-initialize", logger);
cta::objectstore::Agent ag(agr.getAgentAddress(), *be);
ag.initialize();
cta::objectstore::EntryLogSerDeser el("user0", "systemhost", time(NULL));
cta::log::LogContext lc(logger);
re.addOrGetAgentRegisterPointerAndCommit(agr,el, lc);
rel.release();
ag.insertAndRegisterSelf(lc);
......
......@@ -25,6 +25,8 @@
#include "common/Configuration.hpp"
#include "BackendVFS.hpp"
#include "BackendFactory.hpp"
#include "common/log/DummyLogger.hpp"
#include "common/log/LogContext.hpp"
#include "RootEntry.hpp"
#include "Agent.hpp"
#include <iostream>
......@@ -32,12 +34,13 @@
int main(int argc, char ** argv) {
try {
cta::log::DummyLogger dl("");
std::unique_ptr<cta::objectstore::Backend> be;
if (1 == argc) {
cta::common::Configuration m_ctaConf("/etc/cta/cta-frontend.conf");
be = std::move(cta::objectstore::BackendFactory::createBackend(m_ctaConf.getConfEntString("ObjectStore", "BackendPath", nullptr)));
be = std::move(cta::objectstore::BackendFactory::createBackend(m_ctaConf.getConfEntString("ObjectStore", "BackendPath", nullptr), dl));
} else if (2 == argc) {
be.reset(cta::objectstore::BackendFactory::createBackend(argv[1]).release());
be.reset(cta::objectstore::BackendFactory::createBackend(argv[1], dl).release());
} else {
throw std::runtime_error("Wrong number of arguments: expected 0 or 1: [objectstoreURL]");
}
......
......@@ -24,6 +24,8 @@
#include "common/Configuration.hpp"
#include "BackendFactory.hpp"
#include "common/log/StdoutLogger.hpp"
#include "common/log/LogContext.hpp"
#include "BackendVFS.hpp"
#include "Agent.hpp"
#include "RootEntry.hpp"
......@@ -34,14 +36,15 @@
int main(int argc, char ** argv) {
try {
cta::log::StdoutLogger logger("cta-objectstore-unfollow-agent");
std::string agentName;
std::unique_ptr<cta::objectstore::Backend> be;
if (3 == argc) {
be.reset(cta::objectstore::BackendFactory::createBackend(argv[1]).release());
be.reset(cta::objectstore::BackendFactory::createBackend(argv[1], logger).release());
agentName = argv[2];
} else if (2 == argc) {
cta::common::Configuration m_ctaConf("/etc/cta/cta-frontend.conf");
be=std::move(cta::objectstore::BackendFactory::createBackend(m_ctaConf.getConfEntString("ObjectStore", "BackendPath", nullptr)));
be=std::move(cta::objectstore::BackendFactory::createBackend(m_ctaConf.getConfEntString("ObjectStore", "BackendPath", nullptr), logger));
agentName = argv[1];
} else {
throw std::runtime_error("Wrong number of arguments: expected 1 or 2: [objectstoreURL] agentName");
......
......@@ -1728,7 +1728,7 @@ const SchedulerDatabase::ArchiveMount::MountInfo& OStoreDB::ArchiveMount::getMou
//------------------------------------------------------------------------------
std::list<std::unique_ptr<SchedulerDatabase::ArchiveJob> > OStoreDB::ArchiveMount::getNextJobBatch(uint64_t filesRequested,
uint64_t bytesRequested, log::LogContext& logContext) {
utils::Timer t;
utils::Timer t, totalTime;
double driveRegisterCheckTime = 0;
double findQueueTime = 0;
double lockFetchQueueTime = 0;
......@@ -2074,7 +2074,8 @@ std::list<std::unique_ptr<SchedulerDatabase::ArchiveJob> > OStoreDB::ArchiveMoun
.add("jobsUpdateTime", jobsUpdateTime)
.add("queueProcessTime", queueProcessTime)
.add("ownershipRemovalTime", ownershipRemovalTime)
.add("queueCommitTime", queueCommitTime);
.add("queueCommitTime", queueCommitTime)
.add("schedulerDbTime", totalTime.secs());
logContext.log(log::INFO, "In ArchiveMount::getNextJobBatch(): jobs retrieval complete.");
}
// We can construct the return value.
......@@ -2283,7 +2284,7 @@ std::list<std::unique_ptr<SchedulerDatabase::RetrieveJob> > OStoreDB::RetrieveMo
} catch (cta::exception::Exception & e) {
std::string debugType=typeid(e).name();
if (typeid(e) == typeid(Backend::NoSuchObject) ||
typeid(e) == typeid(objectstore::ArchiveRequest::WrongPreviousOwner)) {
typeid(e) == typeid(objectstore::RetrieveRequest::WrongPreviousOwner)) {
// The object was not present or not owned, so we skip it. It should be removed from
// the queue.
jobsToDequeue.emplace_back((*j)->m_retrieveRequest.getAddressIfSet());
......
......@@ -165,8 +165,8 @@ public:
}
private:
std::unique_ptr <cta::objectstore::Backend> m_backend;
std::unique_ptr <cta::log::Logger> m_logger;
std::unique_ptr <cta::objectstore::Backend> m_backend;
std::unique_ptr <cta::catalogue::Catalogue> m_catalogue;
cta::OStoreDB m_OStoreDB;
objectstore::AgentReference m_agentReference;
......@@ -175,7 +175,7 @@ private:
template <>
OStoreDBWrapper<cta::objectstore::BackendVFS>::OStoreDBWrapper(
const std::string &context, const std::string &URL) :
m_backend(new cta::objectstore::BackendVFS()), m_logger(new cta::log::DummyLogger("")),
m_logger(new cta::log::DummyLogger("")), m_backend(new cta::objectstore::BackendVFS()),
m_catalogue(new cta::catalogue::DummyCatalogue(*m_logger)),
m_OStoreDB(*m_backend, *m_catalogue, *m_logger), m_agentReference("OStoreDBFactory", *m_logger) {
// We need to populate the root entry before using.
......@@ -202,7 +202,7 @@ m_OStoreDB(*m_backend, *m_catalogue, *m_logger), m_agentReference("OStoreDBFacto