From 8733c234b5a3dbd3e5bc5e5bd0ce34ec51e8536e Mon Sep 17 00:00:00 2001
From: Cedric CAFFY <cedric.caffy@cern.ch>
Date: Fri, 6 Dec 2019 13:58:44 +0100
Subject: [PATCH] Appended the oss.asize parameter to file dstURL for Retrieval
---
common/dataStructures/RetrieveRequest.cpp | 6 ++++++
common/dataStructures/RetrieveRequest.hpp | 7 +++++++
common/utils/UtilsTest.cpp | 23 +++++++++++++++++++++++
common/utils/utils.cpp | 20 ++++++++++++++++++++
common/utils/utils.hpp | 13 +++++++++++++
scheduler/OStoreDB/OStoreDB.cpp | 5 +++++
6 files changed, 74 insertions(+)
diff --git a/common/dataStructures/RetrieveRequest.cpp b/common/dataStructures/RetrieveRequest.cpp
index 43d170d5de..405e96e93b 100644
--- a/common/dataStructures/RetrieveRequest.cpp
+++ b/common/dataStructures/RetrieveRequest.cpp
@@ -19,6 +19,7 @@
#include "common/dataStructures/RetrieveRequest.hpp"
#include "common/dataStructures/utils.hpp"
#include "common/exception/Exception.hpp"
+#include "common/utils/utils.hpp"
namespace cta {
namespace common {
@@ -59,6 +60,11 @@ std::ostream &operator<<(std::ostream &os, const RetrieveRequest &obj) {
return os;
}
+
+void RetrieveRequest::appendFileSizeToDstURL(const uint64_t fileSize) {
+ cta::utils::appendParameterXRootFileURL(dstURL,"oss.asize",std::to_string(fileSize));
+}
+
} // namespace dataStructures
} // namespace common
} // namespace cta
diff --git a/common/dataStructures/RetrieveRequest.hpp b/common/dataStructures/RetrieveRequest.hpp
index ae4b691fbf..3d68fd7403 100644
--- a/common/dataStructures/RetrieveRequest.hpp
+++ b/common/dataStructures/RetrieveRequest.hpp
@@ -44,6 +44,13 @@ struct RetrieveRequest {
bool operator==(const RetrieveRequest &rhs) const;
bool operator!=(const RetrieveRequest &rhs) const;
+
+ /**
+ * Idempotently append the fileSize to the dstURL
+ * The fileSize will be append only if the dstURL is an XRootD one
+ * @param fileSize the file size to append
+ */
+ void appendFileSizeToDstURL(const uint64_t fileSize);
RequesterIdentity requester;
uint64_t archiveFileID;
diff --git a/common/utils/UtilsTest.cpp b/common/utils/UtilsTest.cpp
index 9166f0ce93..1b0497fffc 100644
--- a/common/utils/UtilsTest.cpp
+++ b/common/utils/UtilsTest.cpp
@@ -863,4 +863,27 @@ TEST_F(cta_UtilsTest, searchAndReplace) {
ASSERT_EQ("one replacement three four one replacement three four", str);
}
+TEST_F(cta_UtilsTest, appendParameterXRootFileURL){
+ std::string fileURLTest = "root://ctaeos.cta.svc.cluster.local//eos/ctaeos/preprod/79fe26de-6b8b-437c-b507-06dbfe8d0a79/0/test00000171?eos.lfn=fxid:b2&eos.ruid=0&eos.rgid=0&eos.injection=1&eos.workflow=retrieve_written&eos.space=default&oss.asize=15360";
+ std::string fileURL = fileURLTest;
+ cta::utils::appendParameterXRootFileURL(fileURL,"oss.asize","145");
+ //nothing should have changed
+ ASSERT_EQ(fileURLTest,fileURL);
+
+ fileURLTest = "root://ctaeos.cta.svc.cluster.local//eos/ctaeos/preprod/79fe26de-6b8b-437c-b507-06dbfe8d0a79/0/test00000171";
+ fileURL = fileURLTest;
+ cta::utils::appendParameterXRootFileURL(fileURL,"oss.asize","15360");
+ ASSERT_EQ(fileURLTest+"?oss.asize=15360",fileURL);
+
+ fileURLTest = "file://path_to_folder/path_to_file";
+ fileURL = fileURLTest;
+ cta::utils::appendParameterXRootFileURL(fileURL,"oss.asize","15360");
+ ASSERT_EQ(fileURLTest,fileURL);
+
+ fileURLTest = "root://ctaeos.cta.svc.cluster.local//eos/ctaeos/preprod/79fe26de-6b8b-437c-b507-06dbfe8d0a79/0/test00000171?eos.lfn=fxid:b2&eos.ruid=0&eos.rgid=0&eos.injection=1&eos.workflow=retrieve_written&eos.space=default";
+ fileURL = fileURLTest;
+ cta::utils::appendParameterXRootFileURL(fileURL,"oss.asize","15360");
+ ASSERT_EQ(fileURLTest+"&oss.asize=15360",fileURL);
+}
+
} // namespace unitTests
diff --git a/common/utils/utils.cpp b/common/utils/utils.cpp
index 5f538c6955..642e8a891c 100644
--- a/common/utils/utils.cpp
+++ b/common/utils/utils.cpp
@@ -940,5 +940,25 @@ void segfault() {
*((int *)nullptr) = 0;
}
+//------------------------------------------------------------------------------
+// appendParameterXRootFileURL
+//------------------------------------------------------------------------------
+void appendParameterXRootFileURL(std::string &fileURL, const std::string ¶meterName, const std::string &value){
+ cta::utils::Regex regexXrootFile("^(root://.*)$");
+ if(regexXrootFile.exec(fileURL).size()){
+ std::string parameterToAppend = parameterName+"="+value;
+ if(fileURL.find("?") == std::string::npos){
+ //No parameter at the end of the XRootd fileURL
+ fileURL.append("?"+parameterToAppend);
+ return;
+ }
+ //There are parameters in the fileURL, check if the parameter is
+ //there or not
+ if(fileURL.find("&"+parameterName) == std::string::npos){
+ fileURL.append("&"+parameterToAppend);
+ }
+ }
+}
+
} // namespace utils
} // namespace cta
diff --git a/common/utils/utils.hpp b/common/utils/utils.hpp
index 63bc8e66ca..2968010d06 100644
--- a/common/utils/utils.hpp
+++ b/common/utils/utils.hpp
@@ -25,6 +25,8 @@
#include <unistd.h>
#include <vector>
+#include "Regex.hpp"
+
namespace cta {
namespace utils {
@@ -422,6 +424,17 @@ namespace utils {
* Here, the CPU will interrupt the process with a memory protection error.
*/
void segfault(void);
+
+ /**
+ * Idempotently append the XRootD parameter to the fileURL
+ * If the fileURL is not an XRootD path (does not start with root://),
+ * the fileURL will not be changed.
+ * If the fileURL already has this parameter, it will not be changed.
+ * @param fileURL [input, output]
+ * @param parameterName [input]
+ * @param value [input]
+ */
+ void appendParameterXRootFileURL(std::string &fileURL, const std::string ¶meterName, const std::string &value);
} // namespace utils
diff --git a/scheduler/OStoreDB/OStoreDB.cpp b/scheduler/OStoreDB/OStoreDB.cpp
index 90ecf2377f..f34efc0eb7 100644
--- a/scheduler/OStoreDB/OStoreDB.cpp
+++ b/scheduler/OStoreDB/OStoreDB.cpp
@@ -1185,6 +1185,10 @@ SchedulerDatabase::RetrieveRequestInfo OStoreDB::queueRetrieve(cta::common::data
for (auto & tf: criteria.archiveFile.tapeFiles) {
if (tf.vid == ret.selectedVid) {
bestCopyNb = tf.copyNb;
+ // Appending the file size to the dstURL so that
+ // XrootD will fail to retrieve if there is not enough free space
+ // in the eos disk
+ rqst.appendFileSizeToDstURL(tf.fileSize);
goto vidFound;
}
}
@@ -2339,6 +2343,7 @@ uint64_t OStoreDB::RepackRequest::addSubrequestsAndUpdateStats(std::list<Subrequ
common::dataStructures::RetrieveRequest schedReq;
schedReq.archiveFileID = rsr.archiveFile.archiveFileID;
schedReq.dstURL = rsr.fileBufferURL;
+ schedReq.appendFileSizeToDstURL(rsr.archiveFile.fileSize);
schedReq.diskFileInfo = rsr.archiveFile.diskFileInfo;
// dsrr.errorReportURL: We leave this bank as the reporting will be done to the repack request,
// stored in the repack info.
--
GitLab