diff --git a/common/dataStructures/RetrieveRequest.cpp b/common/dataStructures/RetrieveRequest.cpp index 43d170d5de4d015bd71c2501caa6afbf28df14a7..405e96e93b2d6392ff3b12addf6708b2fc248aac 100644 --- a/common/dataStructures/RetrieveRequest.cpp +++ b/common/dataStructures/RetrieveRequest.cpp @@ -19,6 +19,7 @@ #include "common/dataStructures/RetrieveRequest.hpp" #include "common/dataStructures/utils.hpp" #include "common/exception/Exception.hpp" +#include "common/utils/utils.hpp" namespace cta { namespace common { @@ -59,6 +60,11 @@ std::ostream &operator<<(std::ostream &os, const RetrieveRequest &obj) { return os; } + +void RetrieveRequest::appendFileSizeToDstURL(const uint64_t fileSize) { + cta::utils::appendParameterXRootFileURL(dstURL,"oss.asize",std::to_string(fileSize)); +} + } // namespace dataStructures } // namespace common } // namespace cta diff --git a/common/dataStructures/RetrieveRequest.hpp b/common/dataStructures/RetrieveRequest.hpp index ae4b691fbf5230ded5df02aca4d587e9c3551c30..3d68fd7403c2287821826972f79176829d85584e 100644 --- a/common/dataStructures/RetrieveRequest.hpp +++ b/common/dataStructures/RetrieveRequest.hpp @@ -44,6 +44,13 @@ struct RetrieveRequest { bool operator==(const RetrieveRequest &rhs) const; bool operator!=(const RetrieveRequest &rhs) const; + + /** + * Idempotently append the fileSize to the dstURL + * The fileSize will be append only if the dstURL is an XRootD one + * @param fileSize the file size to append + */ + void appendFileSizeToDstURL(const uint64_t fileSize); RequesterIdentity requester; uint64_t archiveFileID; diff --git a/common/utils/UtilsTest.cpp b/common/utils/UtilsTest.cpp index 9166f0ce930e429b98796c2e313b045ae9e83cd5..1b0497fffcbf669b01172ecf931f977777f0a905 100644 --- a/common/utils/UtilsTest.cpp +++ b/common/utils/UtilsTest.cpp @@ -863,4 +863,27 @@ TEST_F(cta_UtilsTest, searchAndReplace) { ASSERT_EQ("one replacement three four one replacement three four", str); } +TEST_F(cta_UtilsTest, appendParameterXRootFileURL){ + std::string fileURLTest = "root://ctaeos.cta.svc.cluster.local//eos/ctaeos/preprod/79fe26de-6b8b-437c-b507-06dbfe8d0a79/0/test00000171?eos.lfn=fxid:b2&eos.ruid=0&eos.rgid=0&eos.injection=1&eos.workflow=retrieve_written&eos.space=default&oss.asize=15360"; + std::string fileURL = fileURLTest; + cta::utils::appendParameterXRootFileURL(fileURL,"oss.asize","145"); + //nothing should have changed + ASSERT_EQ(fileURLTest,fileURL); + + fileURLTest = "root://ctaeos.cta.svc.cluster.local//eos/ctaeos/preprod/79fe26de-6b8b-437c-b507-06dbfe8d0a79/0/test00000171"; + fileURL = fileURLTest; + cta::utils::appendParameterXRootFileURL(fileURL,"oss.asize","15360"); + ASSERT_EQ(fileURLTest+"?oss.asize=15360",fileURL); + + fileURLTest = "file://path_to_folder/path_to_file"; + fileURL = fileURLTest; + cta::utils::appendParameterXRootFileURL(fileURL,"oss.asize","15360"); + ASSERT_EQ(fileURLTest,fileURL); + + fileURLTest = "root://ctaeos.cta.svc.cluster.local//eos/ctaeos/preprod/79fe26de-6b8b-437c-b507-06dbfe8d0a79/0/test00000171?eos.lfn=fxid:b2&eos.ruid=0&eos.rgid=0&eos.injection=1&eos.workflow=retrieve_written&eos.space=default"; + fileURL = fileURLTest; + cta::utils::appendParameterXRootFileURL(fileURL,"oss.asize","15360"); + ASSERT_EQ(fileURLTest+"&oss.asize=15360",fileURL); +} + } // namespace unitTests diff --git a/common/utils/utils.cpp b/common/utils/utils.cpp index 5f538c695504e12c509d2dff873343af7b2ebece..642e8a891cd3e60de9243f356a8484679ac69501 100644 --- a/common/utils/utils.cpp +++ b/common/utils/utils.cpp @@ -940,5 +940,25 @@ void segfault() { *((int *)nullptr) = 0; } +//------------------------------------------------------------------------------ +// appendParameterXRootFileURL +//------------------------------------------------------------------------------ +void appendParameterXRootFileURL(std::string &fileURL, const std::string ¶meterName, const std::string &value){ + cta::utils::Regex regexXrootFile("^(root://.*)$"); + if(regexXrootFile.exec(fileURL).size()){ + std::string parameterToAppend = parameterName+"="+value; + if(fileURL.find("?") == std::string::npos){ + //No parameter at the end of the XRootd fileURL + fileURL.append("?"+parameterToAppend); + return; + } + //There are parameters in the fileURL, check if the parameter is + //there or not + if(fileURL.find("&"+parameterName) == std::string::npos){ + fileURL.append("&"+parameterToAppend); + } + } +} + } // namespace utils } // namespace cta diff --git a/common/utils/utils.hpp b/common/utils/utils.hpp index 63bc8e66ca03ad1aee8d202c1a6983d3d2bffbb2..2968010d06b2e8c8591ede8e7855d9938eac9796 100644 --- a/common/utils/utils.hpp +++ b/common/utils/utils.hpp @@ -25,6 +25,8 @@ #include <unistd.h> #include <vector> +#include "Regex.hpp" + namespace cta { namespace utils { @@ -422,6 +424,17 @@ namespace utils { * Here, the CPU will interrupt the process with a memory protection error. */ void segfault(void); + + /** + * Idempotently append the XRootD parameter to the fileURL + * If the fileURL is not an XRootD path (does not start with root://), + * the fileURL will not be changed. + * If the fileURL already has this parameter, it will not be changed. + * @param fileURL [input, output] + * @param parameterName [input] + * @param value [input] + */ + void appendParameterXRootFileURL(std::string &fileURL, const std::string ¶meterName, const std::string &value); } // namespace utils diff --git a/scheduler/OStoreDB/OStoreDB.cpp b/scheduler/OStoreDB/OStoreDB.cpp index 90ecf2377f687ac06cc89869474f13a36c6ded06..f34efc0eb78fd424d7e1eae13ff17b003f4bbf7f 100644 --- a/scheduler/OStoreDB/OStoreDB.cpp +++ b/scheduler/OStoreDB/OStoreDB.cpp @@ -1185,6 +1185,10 @@ SchedulerDatabase::RetrieveRequestInfo OStoreDB::queueRetrieve(cta::common::data for (auto & tf: criteria.archiveFile.tapeFiles) { if (tf.vid == ret.selectedVid) { bestCopyNb = tf.copyNb; + // Appending the file size to the dstURL so that + // XrootD will fail to retrieve if there is not enough free space + // in the eos disk + rqst.appendFileSizeToDstURL(tf.fileSize); goto vidFound; } } @@ -2339,6 +2343,7 @@ uint64_t OStoreDB::RepackRequest::addSubrequestsAndUpdateStats(std::list<Subrequ common::dataStructures::RetrieveRequest schedReq; schedReq.archiveFileID = rsr.archiveFile.archiveFileID; schedReq.dstURL = rsr.fileBufferURL; + schedReq.appendFileSizeToDstURL(rsr.archiveFile.fileSize); schedReq.diskFileInfo = rsr.archiveFile.diskFileInfo; // dsrr.errorReportURL: We leave this bank as the reporting will be done to the repack request, // stored in the repack info.