From 161764874e5de4e416b661a37ee1a6e0c155f8dd Mon Sep 17 00:00:00 2001
From: Michael Davis <michael.davis@cern.ch>
Date: Thu, 30 Jan 2020 14:41:29 +0100
Subject: [PATCH] [frontend] Adds configurable maximum file size limit
cta.archivefile.max_size_gb 1000
Specifies maximum file size that the CTA Frontend will accept to queue
for archival. Defaults to 2 TB and can be configured in in increments
of 1 GB.
---
xroot_plugins/XrdSsiCtaRequestMessage.cpp | 9 +++++++--
xroot_plugins/XrdSsiCtaRequestMessage.hpp | 2 ++
xroot_plugins/XrdSsiCtaServiceProvider.cpp | 7 ++++++-
xroot_plugins/XrdSsiCtaServiceProvider.hpp | 8 +++++++-
xroot_plugins/cta-frontend-xrootd.conf | 3 +++
5 files changed, 25 insertions(+), 4 deletions(-)
diff --git a/xroot_plugins/XrdSsiCtaRequestMessage.cpp b/xroot_plugins/XrdSsiCtaRequestMessage.cpp
index fe4681d5c2..c3e5a02134 100644
--- a/xroot_plugins/XrdSsiCtaRequestMessage.cpp
+++ b/xroot_plugins/XrdSsiCtaRequestMessage.cpp
@@ -393,11 +393,17 @@ void RequestMessage::processCLOSEW(const cta::eos::Notification ¬ification, c
throw PbException(std::string(__FUNCTION__) + ": Failed to find the extended attribute named sys.archive.storage_class");
}
- // For testing, this storage class will always fail
+ // For testing: this storage class will always fail
if(storageClassItor->second == "fail_on_closew_test") {
throw PbException("File is in fail_on_closew_test storage class, which always fails.");
}
+ // Disallow archival of files above the specified limit
+ if(notification.file().size() > m_archiveFileMaxSize) {
+ throw exception::UserError("Archive request rejected: file size (" + std::to_string(notification.file().size()) +
+ " bytes) exceeds maximum allowed size (" + std::to_string(m_archiveFileMaxSize) + " bytes)");
+ }
+
cta::common::dataStructures::ArchiveRequest request;
checksum::ProtobufToChecksumBlob(notification.file().csb(), request.checksumBlob);
request.diskFileInfo.owner_uid = notification.file().owner().uid();
@@ -417,7 +423,6 @@ void RequestMessage::processCLOSEW(const cta::eos::Notification ¬ification, c
// CTA Archive ID is an EOS extended attribute, i.e. it is stored as a string, which
// must be converted to a valid uint64_t
-
const auto archiveFileIdItor = notification.file().xattr().find("sys.archive.file_id");
if(notification.file().xattr().end() == archiveFileIdItor) {
throw PbException(std::string(__FUNCTION__) + ": Failed to find the extended attribute named sys.archive.file_id");
diff --git a/xroot_plugins/XrdSsiCtaRequestMessage.hpp b/xroot_plugins/XrdSsiCtaRequestMessage.hpp
index 67bf296858..c71640b750 100644
--- a/xroot_plugins/XrdSsiCtaRequestMessage.hpp
+++ b/xroot_plugins/XrdSsiCtaRequestMessage.hpp
@@ -36,6 +36,7 @@ public:
m_scheddb(service->getSchedDb()),
m_catalogue(service->getCatalogue()),
m_scheduler(service->getScheduler()),
+ m_archiveFileMaxSize(service->getArchiveFileMaxSize()),
m_repackBufferURL(service->getRepackBufferURL()),
m_namespaceMap(service->getNamespaceMap()),
m_lc(service->getLogContext()) {
@@ -271,6 +272,7 @@ private:
cta::OStoreDBWithAgent &m_scheddb; //!< Reference to CTA ObjectStore
cta::catalogue::Catalogue &m_catalogue; //!< Reference to CTA Catalogue
cta::Scheduler &m_scheduler; //!< Reference to CTA Scheduler
+ uint64_t m_archiveFileMaxSize; //!< Maximum allowed file size for archive requests
optional<std::string> m_repackBufferURL; //!< Repack buffer URL
NamespaceMap_t m_namespaceMap; //!< Identifiers for namespace queries
cta::log::LogContext m_lc; //!< CTA Log Context
diff --git a/xroot_plugins/XrdSsiCtaServiceProvider.cpp b/xroot_plugins/XrdSsiCtaServiceProvider.cpp
index ee42e84d96..bb2f58de6a 100644
--- a/xroot_plugins/XrdSsiCtaServiceProvider.cpp
+++ b/xroot_plugins/XrdSsiCtaServiceProvider.cpp
@@ -148,7 +148,12 @@ void XrdSsiCtaServiceProvider::ExceptionThrowingInit(XrdSsiLogger *logP, XrdSsiC
} catch (std::bad_cast &) {
// If not, never mind
}
-
+
+ // Initialise the Frontend
+ auto archiveFileMaxSize = config.getOptionValueInt("cta.archivefile.max_size_gb");
+ m_archiveFileMaxSize = archiveFileMaxSize.first ? archiveFileMaxSize.second : 2048; // GB
+ m_archiveFileMaxSize *= 1024*1024*1024; // bytes
+
// Get the repack buffer URL
auto repackBufferURLConf = config.getOptionValueStr("cta.repack.repack_buffer_url");
if(repackBufferURLConf.first){
diff --git a/xroot_plugins/XrdSsiCtaServiceProvider.hpp b/xroot_plugins/XrdSsiCtaServiceProvider.hpp
index 74acb3ce53..f4fa5da741 100644
--- a/xroot_plugins/XrdSsiCtaServiceProvider.hpp
+++ b/xroot_plugins/XrdSsiCtaServiceProvider.hpp
@@ -104,7 +104,12 @@ public:
* Get the log context for this Service
*/
cta::log::LogContext getLogContext() const { return cta::log::LogContext(*m_log); }
-
+
+ /*!
+ * Get the maximum file size for an archive request
+ */
+ uint64_t getArchiveFileMaxSize() const { return m_archiveFileMaxSize; }
+
/*!
* Get the repack buffer URL
*/
@@ -148,6 +153,7 @@ private:
// Member variables
+ uint64_t m_archiveFileMaxSize; //!< Maximum allowed file size for archive requests
std::unique_ptr<cta::objectstore::Backend> m_backend; //!< VFS backend for the objectstore DB
std::unique_ptr<cta::objectstore::BackendPopulator> m_backendPopulator; //!< Object used to populate the backend
std::unique_ptr<cta::OStoreDBWithAgent> m_scheddb; //!< DB/Object Store of persistent objects
diff --git a/xroot_plugins/cta-frontend-xrootd.conf b/xroot_plugins/cta-frontend-xrootd.conf
index b7b88c1430..9ac9bc8d3d 100644
--- a/xroot_plugins/cta-frontend-xrootd.conf
+++ b/xroot_plugins/cta-frontend-xrootd.conf
@@ -11,6 +11,9 @@ cta.schedulerdb.numberofthreads 500
# CTA Catalogue options
cta.catalogue.numberofconnections 10
+# Maximum file size (in GB) that the CTA Frontend will accept for archiving
+cta.archivefile.max_size_gb 1000
+
# CTA Frontend log URL
cta.log.url file:/var/log/cta/cta-frontend.log
--
GitLab