diff --git a/catalogue/CMakeLists.txt b/catalogue/CMakeLists.txt
index d43302c42586fd30d5ed1f0e85f8f9329259a5ac..3e6cd3b5176bf653d2ef4c647b15a93f389b245d 100644
--- a/catalogue/CMakeLists.txt
+++ b/catalogue/CMakeLists.txt
@@ -60,8 +60,15 @@ set (CATALOGUE_LIB_SRC_FILES
UserSpecifiedANonEmptyTape.cpp
UserSpecifiedANonExistentLogicalLibrary.cpp
UserSpecifiedANonExistentTape.cpp
+ UserSpecifiedANonExistentDiskSystem.cpp
+ UserSpecifiedANonEmptyDiskSystemAfterDelete.cpp
UserSpecifiedAnEmptyStringComment.cpp
UserSpecifiedAnEmptyStringDiskInstanceName.cpp
+ UserSpecifiedAnEmptyStringDiskSystemName.cpp
+ UserSpecifiedAnEmptyStringFileRegexp.cpp
+ UserSpecifiedAnEmptyStringFreeSpaceQueryURL.cpp
+ UserSpecifiedAZeroRefreshInterval.cpp
+ UserSpecifiedAZeroTargetedFreeSpace.cpp
UserSpecifiedAnEmptyStringLogicalLibraryName.cpp
UserSpecifiedAnEmptyStringMediaType.cpp
UserSpecifiedAnEmptyStringStorageClassName.cpp
@@ -171,7 +178,7 @@ set_property(TARGET ctainmemorycatalogueunittests PROPERTY SOVERSION "${CTA_SOVE
set_property(TARGET ctainmemorycatalogueunittests PROPERTY VERSION "${CTA_LIBVERSION}")
target_link_libraries (ctainmemorycatalogueunittests
- ctacatalogue)
+ ctacatalogue ctadisk)
install (TARGETS ctainmemorycatalogueunittests DESTINATION usr/${CMAKE_INSTALL_LIBDIR})
diff --git a/catalogue/Catalogue.hpp b/catalogue/Catalogue.hpp
index 305f75ce995cb64863e0d80d67ff819164273a0c..5424fa3d859879221e0d7a052d2ff07db9ad1cf6 100644
--- a/catalogue/Catalogue.hpp
+++ b/catalogue/Catalogue.hpp
@@ -55,6 +55,7 @@
#include "common/dataStructures/RequesterIdentity.hpp"
#include "common/dataStructures/VidToTapeMap.hpp"
#include "common/dataStructures/WriteTestResult.hpp"
+#include "disk/DiskSystem.hpp"
#include "common/exception/FileSizeMismatch.hpp"
#include "common/exception/TapeFseqMismatch.hpp"
#include "common/exception/UserError.hpp"
@@ -541,6 +542,60 @@ public:
virtual void deleteActivitiesFairShareWeight(const common::dataStructures::SecurityIdentity &admin, const std::string & diskInstanceName, const std::string & acttivity) = 0;
virtual std::list<common::dataStructures::ActivitiesFairShareWeights> getActivitiesFairShareWeights() const = 0;
+ /**
+ * Returns all the disk systems within the CTA catalogue.
+ *
+ * @return The disk systems.
+ * requester group.
+ */
+ virtual disk::DiskSystemList getAllDiskSystems() const = 0;
+
+ /**
+ * Creates a disk system.
+ *
+ * @param admin The administrator.
+ * @param name The name of the disk system.
+ * @param fileRegexp The regular expression allowing matching destination URLs
+ * for this disk system.
+ * @param freeSpaceQueryURL The query URL that describes a method to query the
+ * free space from the disk system.
+ * @param refreshInterval The refresh interval (seconds) defining how long do
+ * we use a free space value.
+ * @param targetedFreeSpace The targeted free space (margin) based on the free
+ * space update latency (inherent to the file system and induced by the refresh
+ * interval), and the expected external bandwidth from sources external to CTA.
+ * @param comment Comment.
+ */
+ virtual void createDiskSystem(
+ const common::dataStructures::SecurityIdentity &admin,
+ const std::string &name,
+ const std::string &fileRegexp,
+ const std::string &freeSpaceQueryURL,
+ const uint64_t refreshInterval,
+ const uint64_t targetedFreeSpace,
+ const uint64_t sleepTime,
+ const std::string &comment) = 0;
+
+ /**
+ * Deletes a disk system.
+ *
+ * @param name The name of the disk system.
+ */
+ virtual void deleteDiskSystem(const std::string &name) = 0;
+
+ virtual void modifyDiskSystemFileRegexp(const common::dataStructures::SecurityIdentity &admin,
+ const std::string &name, const std::string &fileRegexp) = 0;
+ virtual void modifyDiskSystemFreeSpaceQueryURL(const common::dataStructures::SecurityIdentity &admin,
+ const std::string &name, const std::string &freeSpaceQueryURL) = 0;
+ virtual void modifyDiskSystemRefreshInterval(const common::dataStructures::SecurityIdentity &admin,
+ const std::string &name, const uint64_t refreshInterval) = 0;
+ virtual void modifyDiskSystemTargetedFreeSpace(const common::dataStructures::SecurityIdentity &admin,
+ const std::string &name, const uint64_t targetedFreeSpace) = 0;
+ virtual void modifyDiskSystemSleepTime(const common::dataStructures::SecurityIdentity &admin,
+ const std::string &name, const uint64_t sleepTime) = 0;
+ virtual void modifyDiskSystemComment(const common::dataStructures::SecurityIdentity &admin,
+ const std::string &name, const std::string &comment) = 0;
+
/**
* Returns the specified archive files. Please note that the list of files
* is ordered by archive file ID.
@@ -659,11 +714,20 @@ public:
* @return True if the tape exists.
*/
virtual bool tapeExists(const std::string &vid) const = 0;
+
+ /**
+ * Returns true if the specified disk system exists.
+ *
+ * @param name The name identifier of the disk system.
+ * @return True if the tape exists.
+ */
+ virtual bool diskSystemExists(const std::string &name) const = 0;
}; // class Catalogue
CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedAnEmptyStringActivity);
CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedAnOutOfRangeActivityWeight);
+CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedAZeroSleepTime);
} // namespace catalogue
} // namespace cta
diff --git a/catalogue/CatalogueRetryWrapper.hpp b/catalogue/CatalogueRetryWrapper.hpp
index 4e78ae7a54f301030ec8408dfb9dca749e8e8c08..73797d6a3fe197ee565c89febd4d92b3217b45cc 100644
--- a/catalogue/CatalogueRetryWrapper.hpp
+++ b/catalogue/CatalogueRetryWrapper.hpp
@@ -381,7 +381,42 @@ public:
return retryOnLostConnection(m_log, [&]{return m_catalogue->getActivitiesFairShareWeights();}, m_maxTriesToConnect);
}
+ disk::DiskSystemList getAllDiskSystems() const override {
+ return retryOnLostConnection(m_log, [&]{return m_catalogue->getAllDiskSystems();}, m_maxTriesToConnect);
+ }
+
+ void createDiskSystem(const common::dataStructures::SecurityIdentity &admin, const std::string &name, const std::string &fileRegexp, const std::string &freeSpaceQueryURL, const uint64_t refreshInterval, const uint64_t targetedFreeSpace, const uint64_t sleepTime, const std::string &comment) override {
+ return retryOnLostConnection(m_log, [&]{return m_catalogue->createDiskSystem(admin, name, fileRegexp, freeSpaceQueryURL, refreshInterval, targetedFreeSpace, sleepTime, comment);}, m_maxTriesToConnect);
+ }
+
+ void deleteDiskSystem(const std::string &name) override {
+ return retryOnLostConnection(m_log, [&]{return m_catalogue->deleteDiskSystem(name);}, m_maxTriesToConnect);
+ }
+
+ void modifyDiskSystemFileRegexp(const common::dataStructures::SecurityIdentity &admin, const std::string &name, const std::string &fileRegexp) override {
+ return retryOnLostConnection(m_log, [&]{return m_catalogue->modifyDiskSystemFileRegexp(admin, name, fileRegexp);}, m_maxTriesToConnect);
+ }
+ void modifyDiskSystemFreeSpaceQueryURL(const common::dataStructures::SecurityIdentity &admin, const std::string &name, const std::string &freeSpaceQueryURL) override {
+ return retryOnLostConnection(m_log, [&]{return m_catalogue->modifyDiskSystemFreeSpaceQueryURL(admin, name, freeSpaceQueryURL);}, m_maxTriesToConnect);
+ }
+
+ void modifyDiskSystemRefreshInterval(const common::dataStructures::SecurityIdentity &admin, const std::string &name, const uint64_t refreshInterval) override {
+ return retryOnLostConnection(m_log, [&]{return m_catalogue->modifyDiskSystemRefreshInterval(admin, name, refreshInterval);}, m_maxTriesToConnect);
+ }
+
+ void modifyDiskSystemTargetedFreeSpace(const common::dataStructures::SecurityIdentity &admin, const std::string &name, const uint64_t targetedFreeSpace) override {
+ return retryOnLostConnection(m_log, [&]{return m_catalogue->modifyDiskSystemTargetedFreeSpace(admin, name, targetedFreeSpace);}, m_maxTriesToConnect);
+ }
+
+ void modifyDiskSystemComment(const common::dataStructures::SecurityIdentity &admin, const std::string &name, const std::string &comment) override {
+ return retryOnLostConnection(m_log, [&]{return m_catalogue->modifyDiskSystemComment(admin, name, comment);}, m_maxTriesToConnect);
+ }
+
+ void modifyDiskSystemSleepTime(const common::dataStructures::SecurityIdentity& admin, const std::string& name, const uint64_t sleepTime) override {
+ return retryOnLostConnection(m_log, [&]{return m_catalogue->modifyDiskSystemSleepTime(admin, name, sleepTime);}, m_maxTriesToConnect);
+ }
+
ArchiveFileItor getArchiveFilesItor(const TapeFileSearchCriteria &searchCriteria = TapeFileSearchCriteria()) const override {
return retryOnLostConnection(m_log, [&]{return m_catalogue->getArchiveFilesItor(searchCriteria);}, m_maxTriesToConnect);
}
@@ -426,6 +461,10 @@ public:
return retryOnLostConnection(m_log, [&]{return m_catalogue->tapeExists(vid);}, m_maxTriesToConnect);
}
+ bool diskSystemExists(const std::string &name) const override {
+ return retryOnLostConnection(m_log, [&]{return m_catalogue->diskSystemExists(name);}, m_maxTriesToConnect);
+ };
+
protected:
/**
diff --git a/catalogue/CatalogueTest.cpp b/catalogue/CatalogueTest.cpp
index 9fcf23f773b1c6e9ed4fcef9a3829e439aadaada..106b23ad10f916372890284c312909fc0d350e59 100644
--- a/catalogue/CatalogueTest.cpp
+++ b/catalogue/CatalogueTest.cpp
@@ -22,6 +22,12 @@
#include "catalogue/UserSpecifiedANonEmptyTape.hpp"
#include "catalogue/UserSpecifiedANonExistentLogicalLibrary.hpp"
#include "catalogue/UserSpecifiedANonExistentTape.hpp"
+#include "catalogue/UserSpecifiedANonExistentDiskSystem.hpp"
+#include "catalogue/UserSpecifiedAnEmptyStringDiskSystemName.hpp"
+#include "catalogue/UserSpecifiedAnEmptyStringFileRegexp.hpp"
+#include "catalogue/UserSpecifiedAnEmptyStringFreeSpaceQueryURL.hpp"
+#include "catalogue/UserSpecifiedAZeroRefreshInterval.hpp"
+#include "catalogue/UserSpecifiedAZeroTargetedFreeSpace.hpp"
#include "catalogue/UserSpecifiedAnEmptyStringComment.hpp"
#include "catalogue/UserSpecifiedAnEmptyStringDiskInstanceName.hpp"
#include "catalogue/UserSpecifiedAnEmptyStringLogicalLibraryName.hpp"
@@ -169,6 +175,12 @@ void cta_catalogue_CatalogueTest::SetUp() {
m_catalogue->deleteMountPolicy(mountPolicy.name);
}
}
+ {
+ const auto diskSystems = m_catalogue->getAllDiskSystems();
+ for(auto &ds: diskSystems) {
+ m_catalogue->deleteDiskSystem(ds.name);
+ }
+ }
} catch(exception::Exception &ex) {
throw exception::Exception(std::string(__FUNCTION__) + " failed: " + ex.getMessage().str());
}
@@ -12532,6 +12544,925 @@ TEST_P(cta_catalogue_CatalogueTest, getAllTapes_many_tapes) {
}
}
+TEST_P(cta_catalogue_CatalogueTest, getAllDiskSystems_no_systems) {
+ using namespace cta;
+
+ ASSERT_TRUE(m_catalogue->getAllDiskSystems().empty());
+}
+
+TEST_P(cta_catalogue_CatalogueTest, getAllDiskSystems_many_diskSystems) {
+ using namespace cta;
+
+ ASSERT_TRUE(m_catalogue->getAllDiskSystems().empty());
+
+ const std::string fileRegexp = "file_regexp";
+ const std::string freeSpaceQueryURL = "free_space_query_URL";
+ const uint64_t refreshInterval = 32;
+ const uint64_t targetedFreeSpace = 64;
+ const uint64_t sleepTime = 15*60;
+
+ const uint32_t nbDiskSystems = 16;
+
+ for(uint32_t i = 0; i < nbDiskSystems; i++) {
+ std::ostringstream name;
+ name << "DiskSystem" << std::setfill('0') << std::setw(5) << i;
+ const std::string diskSystemComment = "Create disk system " + name.str();
+ m_catalogue->createDiskSystem(m_admin, name.str(), fileRegexp,
+ freeSpaceQueryURL, refreshInterval + i, targetedFreeSpace + i, sleepTime + i, diskSystemComment);
+ }
+
+ auto diskSystemsList = m_catalogue->getAllDiskSystems();
+ ASSERT_EQ(nbDiskSystems, diskSystemsList.size());
+
+ for(uint32_t i = 0; i < nbDiskSystems; i++) {
+ std::ostringstream name;
+ name << "DiskSystem" << std::setfill('0') << std::setw(5) << i;
+ const std::string diskSystemComment = "Create disk system " + name.str();
+ ASSERT_NO_THROW(diskSystemsList.at(name.str()));
+ const auto diskSystem = diskSystemsList.at(name.str());
+
+ ASSERT_EQ(name.str(), diskSystem.name);
+ ASSERT_EQ(fileRegexp, diskSystem.fileRegexp);
+ ASSERT_EQ(freeSpaceQueryURL, diskSystem.freeSpaceQueryURL);
+ ASSERT_EQ(refreshInterval + i, diskSystem.refreshInterval );
+ ASSERT_EQ(targetedFreeSpace + i, diskSystem.targetedFreeSpace);
+ ASSERT_EQ(diskSystemComment, diskSystem.comment);
+ }
+}
+
+TEST_P(cta_catalogue_CatalogueTest, diskSystemExists_emptyString) {
+ using namespace cta;
+
+ ASSERT_TRUE(m_catalogue->getAllDiskSystems().empty());
+
+ const std::string name = "";
+
+ ASSERT_THROW(m_catalogue->diskSystemExists(name), exception::Exception);
+}
+
+TEST_P(cta_catalogue_CatalogueTest, createDiskSystem_emptyStringDiskSystemName) {
+ using namespace cta;
+
+ ASSERT_TRUE(m_catalogue->getAllDiskSystems().empty());
+
+ const std::string name = "";
+ const std::string fileRegexp = "file_regexp";
+ const std::string freeSpaceQueryURL = "free_space_query_URL";
+ const uint64_t refreshInterval = 32;
+ const uint64_t targetedFreeSpace = 64;
+ const uint64_t sleepTime = 15*60;
+ const std::string comment = "Create disk system";
+
+ ASSERT_THROW(m_catalogue->createDiskSystem(m_admin, name, fileRegexp,
+ freeSpaceQueryURL, refreshInterval, targetedFreeSpace, sleepTime, comment),
+ catalogue::UserSpecifiedAnEmptyStringDiskSystemName);
+}
+
+TEST_P(cta_catalogue_CatalogueTest, createDiskSystem_emptyStringFileRegexp) {
+ using namespace cta;
+
+ ASSERT_TRUE(m_catalogue->getAllDiskSystems().empty());
+
+ const std::string name = "disk_system_name";
+ const std::string fileRegexp = "";
+ const std::string freeSpaceQueryURL = "free_space_query_URL";
+ const uint64_t refreshInterval = 32;
+ const uint64_t targetedFreeSpace = 64;
+ const uint64_t sleepTime = 15*60;
+ const std::string comment = "Create disk system";
+
+ ASSERT_THROW(m_catalogue->createDiskSystem(m_admin, name, fileRegexp,
+ freeSpaceQueryURL, refreshInterval, targetedFreeSpace, sleepTime, comment),
+ catalogue::UserSpecifiedAnEmptyStringFileRegexp);
+}
+
+TEST_P(cta_catalogue_CatalogueTest, createDiskSystem_emptyStringFresSpaceQueryURL) {
+ using namespace cta;
+
+ ASSERT_TRUE(m_catalogue->getAllDiskSystems().empty());
+
+ const std::string name = "disk_system_name";
+ const std::string fileRegexp = "file_regexp";
+ const std::string freeSpaceQueryURL = "";
+ const uint64_t refreshInterval = 32;
+ const uint64_t targetedFreeSpace = 64;
+ const uint64_t sleepTime = 15*60;
+ const std::string comment = "Create disk system";
+
+ ASSERT_THROW(m_catalogue->createDiskSystem(m_admin, name, fileRegexp,
+ freeSpaceQueryURL, refreshInterval, targetedFreeSpace, sleepTime, comment),
+ catalogue::UserSpecifiedAnEmptyStringFreeSpaceQueryURL);
+}
+
+
+TEST_P(cta_catalogue_CatalogueTest, createDiskSystem_zeroRefreshInterval) {
+ using namespace cta;
+
+ ASSERT_TRUE(m_catalogue->getAllDiskSystems().empty());
+
+ const std::string name = "disk_system_name";
+ const std::string fileRegexp = "file_regexp";
+ const std::string freeSpaceQueryURL = "free_space_query_url";
+ const uint64_t refreshInterval = 0;
+ const uint64_t targetedFreeSpace = 64;
+ const uint64_t sleepTime = 15*60;
+ const std::string comment = "Create disk system";
+
+ ASSERT_THROW(m_catalogue->createDiskSystem(m_admin, name, fileRegexp,
+ freeSpaceQueryURL, refreshInterval, targetedFreeSpace, sleepTime, comment),
+ catalogue::UserSpecifiedAZeroRefreshInterval);
+}
+
+TEST_P(cta_catalogue_CatalogueTest, createDiskSystem_zeroTargetedFreeSpace) {
+ using namespace cta;
+
+ ASSERT_TRUE(m_catalogue->getAllDiskSystems().empty());
+
+ const std::string name = "disk_system_name";
+ const std::string fileRegexp = "file_regexp";
+ const std::string freeSpaceQueryURL = "free_space_query_url";
+ const uint64_t refreshInterval = 32;
+ const uint64_t targetedFreeSpace = 0;
+ const uint64_t sleepTime = 15*60;
+ const std::string comment = "Create disk system";
+
+ ASSERT_THROW(m_catalogue->createDiskSystem(m_admin, name, fileRegexp,
+ freeSpaceQueryURL, refreshInterval, targetedFreeSpace, sleepTime, comment),
+ catalogue::UserSpecifiedAZeroTargetedFreeSpace);
+}
+
+TEST_P(cta_catalogue_CatalogueTest, createDiskSystem_emptyStringComment) {
+ using namespace cta;
+
+ ASSERT_TRUE(m_catalogue->getAllDiskSystems().empty());
+
+ const std::string name = "disk_system_name";
+ const std::string fileRegexp = "file_regexp";
+ const std::string freeSpaceQueryURL = "free_space_query_url";
+ const uint64_t refreshInterval = 32;
+ const uint64_t targetedFreeSpace = 64;
+ const uint64_t sleepTime = 15*60;
+ const std::string comment = "";
+
+ ASSERT_THROW(m_catalogue->createDiskSystem(m_admin, name, fileRegexp,
+ freeSpaceQueryURL, refreshInterval, targetedFreeSpace, sleepTime, comment),
+ catalogue::UserSpecifiedAnEmptyStringComment);
+}
+
+TEST_P(cta_catalogue_CatalogueTest, createDiskSystem_9_exabytes_targetedFreeSpace) {
+ using namespace cta;
+
+ ASSERT_TRUE(m_catalogue->getAllDiskSystems().empty());
+
+ const std::string name = "disk_system_name";
+ const std::string fileRegexp = "file_regexp";
+ const std::string freeSpaceQueryURL = "free_space_query_url";
+ const uint64_t refreshInterval = 32;
+ // The maximum size of an SQLite integer is a signed 64-bit integer
+ const uint64_t targetedFreeSpace = 9L * 1000 * 1000 * 1000 * 1000 * 1000 * 1000;
+ const uint64_t sleepTime = 15*60;
+ const std::string comment = "disk system comment";
+
+ m_catalogue->createDiskSystem(m_admin, name, fileRegexp,
+ freeSpaceQueryURL, refreshInterval, targetedFreeSpace, sleepTime, comment);
+
+ const auto diskSystemList = m_catalogue->getAllDiskSystems();
+
+ ASSERT_EQ(1, diskSystemList.size());
+
+ {
+ const auto &diskSystem = diskSystemList.front();
+ ASSERT_EQ(name, diskSystem.name);
+ ASSERT_EQ(fileRegexp, diskSystem.fileRegexp);
+ ASSERT_EQ(freeSpaceQueryURL, diskSystem.freeSpaceQueryURL);
+ ASSERT_EQ(refreshInterval, diskSystem.refreshInterval);
+ ASSERT_EQ(targetedFreeSpace, diskSystem.targetedFreeSpace);
+ ASSERT_EQ(sleepTime, diskSystem.sleepTime);
+ ASSERT_EQ(comment, diskSystem.comment);
+
+ const auto creationLog = diskSystem.creationLog;
+ ASSERT_EQ(m_admin.username, creationLog.username);
+ ASSERT_EQ(m_admin.host, creationLog.host);
+
+ const auto lastModificationLog = diskSystem.lastModificationLog;
+ ASSERT_EQ(creationLog, lastModificationLog);
+ }
+}
+
+TEST_P(cta_catalogue_CatalogueTest, createDiskSystem_sleepTimeHandling) {
+ using namespace cta;
+
+ ASSERT_TRUE(m_catalogue->getAllDiskSystems().empty());
+
+ const std::string name = "disk_system_name";
+ const std::string fileRegexp = "file_regexp";
+ const std::string freeSpaceQueryURL = "free_space_query_url";
+ const uint64_t refreshInterval = 32;
+ const uint64_t targetedFreeSpace = 64;
+ const uint64_t sleepTime = 0;
+ const std::string comment = "disk system comment";
+
+ ASSERT_THROW(m_catalogue->createDiskSystem(m_admin, name, fileRegexp,
+ freeSpaceQueryURL, refreshInterval, targetedFreeSpace, sleepTime, comment),
+ catalogue::UserSpecifiedAZeroSleepTime);
+
+ m_catalogue->createDiskSystem(m_admin, name, fileRegexp,
+ freeSpaceQueryURL, refreshInterval, targetedFreeSpace, std::numeric_limits<int64_t>::max(), comment);
+
+ const auto diskSystemList = m_catalogue->getAllDiskSystems();
+
+ ASSERT_EQ(1, diskSystemList.size());
+
+ {
+ const auto &diskSystem = diskSystemList.front();
+ ASSERT_EQ(name, diskSystem.name);
+ ASSERT_EQ(fileRegexp, diskSystem.fileRegexp);
+ ASSERT_EQ(freeSpaceQueryURL, diskSystem.freeSpaceQueryURL);
+ ASSERT_EQ(refreshInterval, diskSystem.refreshInterval);
+ ASSERT_EQ(targetedFreeSpace, diskSystem.targetedFreeSpace);
+ ASSERT_EQ(std::numeric_limits<int64_t>::max(), diskSystem.sleepTime);
+ ASSERT_EQ(comment, diskSystem.comment);
+
+ const auto creationLog = diskSystem.creationLog;
+ ASSERT_EQ(m_admin.username, creationLog.username);
+ ASSERT_EQ(m_admin.host, creationLog.host);
+
+ const auto lastModificationLog = diskSystem.lastModificationLog;
+ ASSERT_EQ(creationLog, lastModificationLog);
+ }
+}
+
+
+TEST_P(cta_catalogue_CatalogueTest, createDiskSystem_same_twice) {
+ using namespace cta;
+
+ ASSERT_TRUE(m_catalogue->getAllDiskSystems().empty());
+
+ const std::string name = "disk_system_name";
+ const std::string fileRegexp = "file_regexp";
+ const std::string freeSpaceQueryURL = "free_space_query_url";
+ const uint64_t refreshInterval = 32;
+ const uint64_t targetedFreeSpace = 64;
+ const uint64_t sleepTime = 15*60;
+ const std::string comment = "disk system comment";
+
+ m_catalogue->createDiskSystem(m_admin, name, fileRegexp,
+ freeSpaceQueryURL, refreshInterval, targetedFreeSpace, sleepTime, comment);
+
+ const auto diskSystemList = m_catalogue->getAllDiskSystems();
+
+ ASSERT_EQ(1, diskSystemList.size());
+ ASSERT_THROW(m_catalogue->createDiskSystem(m_admin, name, fileRegexp,
+ freeSpaceQueryURL, refreshInterval, targetedFreeSpace, sleepTime, comment), exception::UserError);
+
+}
+
+TEST_P(cta_catalogue_CatalogueTest, deleteDiskSystem) {
+ using namespace cta;
+
+ ASSERT_TRUE(m_catalogue->getAllDiskSystems().empty());
+
+ const std::string name = "disk_system_name";
+ const std::string fileRegexp = "file_regexp";
+ const std::string freeSpaceQueryURL = "free_space_query_url";
+ const uint64_t refreshInterval = 32;
+ const uint64_t targetedFreeSpace = 64;
+ const uint64_t sleepTime = 15*60;
+ const std::string comment = "disk system comment";
+
+ m_catalogue->createDiskSystem(m_admin, name, fileRegexp,
+ freeSpaceQueryURL, refreshInterval, targetedFreeSpace, sleepTime, comment);
+
+ const auto diskSystemList = m_catalogue->getAllDiskSystems();
+
+ ASSERT_EQ(1, diskSystemList.size());
+
+ const auto &diskSystem = diskSystemList.front();
+ ASSERT_EQ(name, diskSystem.name);
+ ASSERT_EQ(fileRegexp, diskSystem.fileRegexp);
+ ASSERT_EQ(freeSpaceQueryURL, diskSystem.freeSpaceQueryURL);
+ ASSERT_EQ(refreshInterval, diskSystem.refreshInterval);
+ ASSERT_EQ(targetedFreeSpace, diskSystem.targetedFreeSpace);
+ ASSERT_EQ(comment, diskSystem.comment);
+
+ const auto creationLog = diskSystem.creationLog;
+ ASSERT_EQ(m_admin.username, creationLog.username);
+ ASSERT_EQ(m_admin.host, creationLog.host);
+
+ const auto lastModificationLog = diskSystem.lastModificationLog;
+ ASSERT_EQ(creationLog, lastModificationLog);
+
+ m_catalogue->deleteDiskSystem(diskSystem.name);
+ ASSERT_TRUE(m_catalogue->getAllDiskSystems().empty());
+}
+
+TEST_P(cta_catalogue_CatalogueTest, deleteDiskSystem_non_existant) {
+ using namespace cta;
+
+ ASSERT_TRUE(m_catalogue->getAllDiskSystems().empty());
+ ASSERT_THROW(m_catalogue->deleteDiskSystem("non_exsitant_disk_system"), catalogue::UserSpecifiedANonExistentDiskSystem);
+}
+
+TEST_P(cta_catalogue_CatalogueTest, modifyDiskSystemFileRegexp) {
+ using namespace cta;
+
+ ASSERT_TRUE(m_catalogue->getAllDiskSystems().empty());
+
+ const std::string name = "disk_system_name";
+ const std::string fileRegexp = "file_regexp";
+ const std::string freeSpaceQueryURL = "free_space_query_url";
+ const uint64_t refreshInterval = 32;
+ const uint64_t targetedFreeSpace = 64;
+ const uint64_t sleepTime = 15*60;
+ const std::string comment = "disk system comment";
+
+ m_catalogue->createDiskSystem(m_admin, name, fileRegexp,
+ freeSpaceQueryURL, refreshInterval, targetedFreeSpace, sleepTime, comment);
+
+ {
+ const auto diskSystemList = m_catalogue->getAllDiskSystems();
+ ASSERT_EQ(1, diskSystemList.size());
+
+ const auto &diskSystem = diskSystemList.front();
+ ASSERT_EQ(name, diskSystem.name);
+ ASSERT_EQ(fileRegexp, diskSystem.fileRegexp);
+ ASSERT_EQ(freeSpaceQueryURL, diskSystem.freeSpaceQueryURL);
+ ASSERT_EQ(refreshInterval, diskSystem.refreshInterval);
+ ASSERT_EQ(targetedFreeSpace, diskSystem.targetedFreeSpace);
+ ASSERT_EQ(comment, diskSystem.comment);
+
+ const auto creationLog = diskSystem.creationLog;
+ ASSERT_EQ(m_admin.username, creationLog.username);
+ ASSERT_EQ(m_admin.host, creationLog.host);
+
+ const auto lastModificationLog = diskSystem.lastModificationLog;
+ ASSERT_EQ(creationLog, lastModificationLog);
+ }
+
+ const std::string modifiedFileRegexp = "modified_fileRegexp";
+ m_catalogue->modifyDiskSystemFileRegexp(m_admin, name, modifiedFileRegexp);
+
+ {
+ const auto diskSystemList = m_catalogue->getAllDiskSystems();
+ ASSERT_EQ(1, diskSystemList.size());
+
+ const auto &diskSystem = diskSystemList.front();
+ ASSERT_EQ(name, diskSystem.name);
+ ASSERT_EQ(modifiedFileRegexp, diskSystem.fileRegexp);
+ ASSERT_EQ(freeSpaceQueryURL, diskSystem.freeSpaceQueryURL);
+ ASSERT_EQ(refreshInterval, diskSystem.refreshInterval);
+ ASSERT_EQ(targetedFreeSpace, diskSystem.targetedFreeSpace);
+ ASSERT_EQ(comment, diskSystem.comment);
+
+ const auto creationLog = diskSystem.creationLog;
+ ASSERT_EQ(m_admin.username, creationLog.username);
+ ASSERT_EQ(m_admin.host, creationLog.host);
+ }
+}
+
+TEST_P(cta_catalogue_CatalogueTest, modifyDiskSystemFileRegexp_emptyStringDiskSystemName) {
+ using namespace cta;
+
+ ASSERT_TRUE(m_catalogue->getAllDiskSystems().empty());
+
+ const std::string diskSystemName = "";
+ const std::string modifiedFileRegexp = "modified_fileRegexp";
+ ASSERT_THROW(m_catalogue->modifyDiskSystemFileRegexp(m_admin, diskSystemName, modifiedFileRegexp),
+ catalogue::UserSpecifiedAnEmptyStringDiskSystemName);
+}
+
+TEST_P(cta_catalogue_CatalogueTest, modifyDiskSystemFileRegexp_nonExistentDiskSystemName) {
+ using namespace cta;
+
+ ASSERT_TRUE(m_catalogue->getAllDiskSystems().empty());
+
+ const std::string diskSystemName = "dummyDiskSystemName";
+ const std::string modifiedFileRegexp = "modified_fileRegexp";
+ ASSERT_THROW(m_catalogue->modifyDiskSystemFileRegexp(m_admin, diskSystemName, modifiedFileRegexp),
+ catalogue::UserSpecifiedANonExistentDiskSystem);
+}
+
+TEST_P(cta_catalogue_CatalogueTest, modifyDiskSystemFileRegexp_emptyStringFileRegexp) {
+ using namespace cta;
+
+ ASSERT_TRUE(m_catalogue->getAllDiskSystems().empty());
+
+ const std::string name = "disk_system_name";
+ const std::string fileRegexp = "file_regexp";
+ const std::string freeSpaceQueryURL = "free_space_query_url";
+ const uint64_t refreshInterval = 32;
+ const uint64_t targetedFreeSpace = 64;
+ const uint64_t sleepTime = 15*60;
+ const std::string comment = "disk system comment";
+
+ m_catalogue->createDiskSystem(m_admin, name, fileRegexp,
+ freeSpaceQueryURL, refreshInterval, targetedFreeSpace, sleepTime, comment);
+
+ {
+ const auto diskSystemList = m_catalogue->getAllDiskSystems();
+ ASSERT_EQ(1, diskSystemList.size());
+
+ const auto &diskSystem = diskSystemList.front();
+ ASSERT_EQ(name, diskSystem.name);
+ ASSERT_EQ(fileRegexp, diskSystem.fileRegexp);
+ ASSERT_EQ(freeSpaceQueryURL, diskSystem.freeSpaceQueryURL);
+ ASSERT_EQ(refreshInterval, diskSystem.refreshInterval);
+ ASSERT_EQ(targetedFreeSpace, diskSystem.targetedFreeSpace);
+ ASSERT_EQ(comment, diskSystem.comment);
+
+ const auto creationLog = diskSystem.creationLog;
+ ASSERT_EQ(m_admin.username, creationLog.username);
+ ASSERT_EQ(m_admin.host, creationLog.host);
+
+ const auto lastModificationLog = diskSystem.lastModificationLog;
+ ASSERT_EQ(creationLog, lastModificationLog);
+ }
+
+ const std::string modifiedFileRegexp = "";
+ ASSERT_THROW(m_catalogue->modifyDiskSystemFileRegexp(m_admin, name, modifiedFileRegexp),
+ catalogue::UserSpecifiedAnEmptyStringFileRegexp);
+}
+
+TEST_P(cta_catalogue_CatalogueTest, modifyDiskSystemFreeSpaceQueryURL) {
+ using namespace cta;
+
+ ASSERT_TRUE(m_catalogue->getAllDiskSystems().empty());
+
+ const std::string name = "disk_system_name";
+ const std::string fileRegexp = "file_regexp";
+ const std::string freeSpaceQueryURL = "free_space_query_url";
+ const uint64_t refreshInterval = 32;
+ const uint64_t targetedFreeSpace = 64;
+ const uint64_t sleepTime = 15*60;
+ const std::string comment = "disk system comment";
+
+ m_catalogue->createDiskSystem(m_admin, name, fileRegexp,
+ freeSpaceQueryURL, refreshInterval, targetedFreeSpace, sleepTime, comment);
+
+ {
+ const auto diskSystemList = m_catalogue->getAllDiskSystems();
+ ASSERT_EQ(1, diskSystemList.size());
+
+ const auto &diskSystem = diskSystemList.front();
+ ASSERT_EQ(name, diskSystem.name);
+ ASSERT_EQ(fileRegexp, diskSystem.fileRegexp);
+ ASSERT_EQ(freeSpaceQueryURL, diskSystem.freeSpaceQueryURL);
+ ASSERT_EQ(refreshInterval, diskSystem.refreshInterval);
+ ASSERT_EQ(targetedFreeSpace, diskSystem.targetedFreeSpace);
+ ASSERT_EQ(comment, diskSystem.comment);
+
+ const auto creationLog = diskSystem.creationLog;
+ ASSERT_EQ(m_admin.username, creationLog.username);
+ ASSERT_EQ(m_admin.host, creationLog.host);
+
+ const auto lastModificationLog = diskSystem.lastModificationLog;
+ ASSERT_EQ(creationLog, lastModificationLog);
+ }
+
+ const std::string modifiedFreeSpaceQueryURL = "modified_freeSpaceQueryURL";
+ m_catalogue->modifyDiskSystemFreeSpaceQueryURL(m_admin, name, modifiedFreeSpaceQueryURL);
+
+ {
+ const auto diskSystemList = m_catalogue->getAllDiskSystems();
+ ASSERT_EQ(1, diskSystemList.size());
+
+ const auto &diskSystem = diskSystemList.front();
+ ASSERT_EQ(name, diskSystem.name);
+ ASSERT_EQ(fileRegexp, diskSystem.fileRegexp);
+ ASSERT_EQ(modifiedFreeSpaceQueryURL, diskSystem.freeSpaceQueryURL);
+ ASSERT_EQ(refreshInterval, diskSystem.refreshInterval);
+ ASSERT_EQ(targetedFreeSpace, diskSystem.targetedFreeSpace);
+ ASSERT_EQ(comment, diskSystem.comment);
+
+ const auto creationLog = diskSystem.creationLog;
+ ASSERT_EQ(m_admin.username, creationLog.username);
+ ASSERT_EQ(m_admin.host, creationLog.host);
+ }
+}
+
+TEST_P(cta_catalogue_CatalogueTest, modifyDiskSystemFreeSpaceQueryURL_emptyStringDiskSystemName) {
+ using namespace cta;
+
+ ASSERT_TRUE(m_catalogue->getAllDiskSystems().empty());
+
+ const std::string diskSystemName = "";
+ const std::string modifiedFreeSpaceQueryURL = "modified_freeSpaceQueryURL";
+ ASSERT_THROW(m_catalogue->modifyDiskSystemFreeSpaceQueryURL(m_admin, diskSystemName, modifiedFreeSpaceQueryURL),
+ catalogue::UserSpecifiedAnEmptyStringDiskSystemName);
+}
+
+TEST_P(cta_catalogue_CatalogueTest, modifyDiskSystemFreeSpaceQueryURL_nonExistentDiskSystemName) {
+ using namespace cta;
+
+ ASSERT_TRUE(m_catalogue->getAllDiskSystems().empty());
+
+ const std::string diskSystemName = "dummyDiskSystemName";
+ const std::string modifiedFreeSpaceQueryURL = "modified_freeSpaceQueryURL";
+ ASSERT_THROW(m_catalogue->modifyDiskSystemFreeSpaceQueryURL(m_admin, diskSystemName, modifiedFreeSpaceQueryURL),
+ catalogue::UserSpecifiedANonExistentDiskSystem);
+}
+
+TEST_P(cta_catalogue_CatalogueTest, modifyDiskSystemFreeSpaceQueryURL_emptyStringFreeSpaceQueryURL) {
+ using namespace cta;
+
+ ASSERT_TRUE(m_catalogue->getAllDiskSystems().empty());
+
+ const std::string name = "disk_system_name";
+ const std::string fileRegexp = "file_regexp";
+ const std::string freeSpaceQueryURL = "free_space_query_url";
+ const uint64_t refreshInterval = 32;
+ const uint64_t targetedFreeSpace = 64;
+ const uint64_t sleepTime = 15*60;
+ const std::string comment = "disk system comment";
+
+ m_catalogue->createDiskSystem(m_admin, name, fileRegexp,
+ freeSpaceQueryURL, refreshInterval, targetedFreeSpace, sleepTime, comment);
+
+ {
+ const auto diskSystemList = m_catalogue->getAllDiskSystems();
+ ASSERT_EQ(1, diskSystemList.size());
+
+ const auto &diskSystem = diskSystemList.front();
+ ASSERT_EQ(name, diskSystem.name);
+ ASSERT_EQ(fileRegexp, diskSystem.fileRegexp);
+ ASSERT_EQ(freeSpaceQueryURL, diskSystem.freeSpaceQueryURL);
+ ASSERT_EQ(refreshInterval, diskSystem.refreshInterval);
+ ASSERT_EQ(targetedFreeSpace, diskSystem.targetedFreeSpace);
+ ASSERT_EQ(comment, diskSystem.comment);
+
+ const auto creationLog = diskSystem.creationLog;
+ ASSERT_EQ(m_admin.username, creationLog.username);
+ ASSERT_EQ(m_admin.host, creationLog.host);
+
+ const auto lastModificationLog = diskSystem.lastModificationLog;
+ ASSERT_EQ(creationLog, lastModificationLog);
+ }
+
+ const std::string modifiedFreeSpaceQueryURL = "";
+ ASSERT_THROW(m_catalogue->modifyDiskSystemFreeSpaceQueryURL(m_admin, name, modifiedFreeSpaceQueryURL),
+ catalogue::UserSpecifiedAnEmptyStringFreeSpaceQueryURL);
+}
+
+TEST_P(cta_catalogue_CatalogueTest, modifyDiskSystemRefreshInterval) {
+ using namespace cta;
+
+ ASSERT_TRUE(m_catalogue->getAllDiskSystems().empty());
+
+ const std::string name = "disk_system_name";
+ const std::string fileRegexp = "file_regexp";
+ const std::string freeSpaceQueryURL = "free_space_query_url";
+ const uint64_t refreshInterval = 32;
+ const uint64_t targetedFreeSpace = 64;
+ const uint64_t sleepTime = 15*60;
+ const std::string comment = "disk system comment";
+
+ m_catalogue->createDiskSystem(m_admin, name, fileRegexp,
+ freeSpaceQueryURL, refreshInterval, targetedFreeSpace, sleepTime, comment);
+
+ {
+ const auto diskSystemList = m_catalogue->getAllDiskSystems();
+ ASSERT_EQ(1, diskSystemList.size());
+
+ const auto &diskSystem = diskSystemList.front();
+ ASSERT_EQ(name, diskSystem.name);
+ ASSERT_EQ(fileRegexp, diskSystem.fileRegexp);
+ ASSERT_EQ(freeSpaceQueryURL, diskSystem.freeSpaceQueryURL);
+ ASSERT_EQ(refreshInterval, diskSystem.refreshInterval);
+ ASSERT_EQ(targetedFreeSpace, diskSystem.targetedFreeSpace);
+ ASSERT_EQ(comment, diskSystem.comment);
+
+ const auto creationLog = diskSystem.creationLog;
+ ASSERT_EQ(m_admin.username, creationLog.username);
+ ASSERT_EQ(m_admin.host, creationLog.host);
+
+ const auto lastModificationLog = diskSystem.lastModificationLog;
+ ASSERT_EQ(creationLog, lastModificationLog);
+ }
+
+ const uint64_t modifiedRefreshInterval = 128;
+ m_catalogue->modifyDiskSystemRefreshInterval(m_admin, name, modifiedRefreshInterval);
+
+ {
+ const auto diskSystemList = m_catalogue->getAllDiskSystems();
+ ASSERT_EQ(1, diskSystemList.size());
+
+ const auto &diskSystem = diskSystemList.front();
+ ASSERT_EQ(name, diskSystem.name);
+ ASSERT_EQ(fileRegexp, diskSystem.fileRegexp);
+ ASSERT_EQ(freeSpaceQueryURL, diskSystem.freeSpaceQueryURL);
+ ASSERT_EQ(modifiedRefreshInterval, diskSystem.refreshInterval);
+ ASSERT_EQ(targetedFreeSpace, diskSystem.targetedFreeSpace);
+ ASSERT_EQ(comment, diskSystem.comment);
+
+ const auto creationLog = diskSystem.creationLog;
+ ASSERT_EQ(m_admin.username, creationLog.username);
+ ASSERT_EQ(m_admin.host, creationLog.host);
+ }
+}
+
+TEST_P(cta_catalogue_CatalogueTest, modifyDiskSystemRefreshInterval_emptyStringDiskSystemName) {
+ using namespace cta;
+
+ ASSERT_TRUE(m_catalogue->getAllDiskSystems().empty());
+
+ const std::string diskSystemName = "";
+ const uint64_t modifiedRefreshInterval = 128;
+ ASSERT_THROW(m_catalogue->modifyDiskSystemRefreshInterval(m_admin, diskSystemName, modifiedRefreshInterval),
+ catalogue::UserSpecifiedAnEmptyStringDiskSystemName);
+}
+
+TEST_P(cta_catalogue_CatalogueTest, modifyDiskSystemRefreshInterval_nonExistentDiskSystemName) {
+ using namespace cta;
+
+ ASSERT_TRUE(m_catalogue->getAllDiskSystems().empty());
+
+ const std::string diskSystemName = "dummyDiskSystemName";
+ const uint64_t modifiedRefreshInterval = 128;
+ ASSERT_THROW(m_catalogue->modifyDiskSystemRefreshInterval(m_admin, diskSystemName, modifiedRefreshInterval),
+ catalogue::UserSpecifiedANonExistentDiskSystem);
+}
+
+TEST_P(cta_catalogue_CatalogueTest, modifyDiskSystemRefreshInterval_zeroRefreshInterval) {
+ using namespace cta;
+
+ ASSERT_TRUE(m_catalogue->getAllDiskSystems().empty());
+
+ const std::string name = "disk_system_name";
+ const std::string fileRegexp = "file_regexp";
+ const std::string freeSpaceQueryURL = "free_space_query_url";
+ const uint64_t refreshInterval = 32;
+ const uint64_t targetedFreeSpace = 64;
+ const uint64_t sleepTime = 15*60;
+ const std::string comment = "disk system comment";
+
+ m_catalogue->createDiskSystem(m_admin, name, fileRegexp,
+ freeSpaceQueryURL, refreshInterval, targetedFreeSpace, sleepTime, comment);
+
+ {
+ const auto diskSystemList = m_catalogue->getAllDiskSystems();
+ ASSERT_EQ(1, diskSystemList.size());
+
+ const auto &diskSystem = diskSystemList.front();
+ ASSERT_EQ(name, diskSystem.name);
+ ASSERT_EQ(fileRegexp, diskSystem.fileRegexp);
+ ASSERT_EQ(freeSpaceQueryURL, diskSystem.freeSpaceQueryURL);
+ ASSERT_EQ(refreshInterval, diskSystem.refreshInterval);
+ ASSERT_EQ(targetedFreeSpace, diskSystem.targetedFreeSpace);
+ ASSERT_EQ(comment, diskSystem.comment);
+
+ const auto creationLog = diskSystem.creationLog;
+ ASSERT_EQ(m_admin.username, creationLog.username);
+ ASSERT_EQ(m_admin.host, creationLog.host);
+
+ const auto lastModificationLog = diskSystem.lastModificationLog;
+ ASSERT_EQ(creationLog, lastModificationLog);
+ }
+
+ const uint64_t modifiedRefreshInterval = 0;
+ ASSERT_THROW(m_catalogue->modifyDiskSystemRefreshInterval(m_admin, name, modifiedRefreshInterval),
+ catalogue::UserSpecifiedAZeroRefreshInterval);
+}
+
+TEST_P(cta_catalogue_CatalogueTest, modifyDiskSystemTargetedFreeSpace) {
+ using namespace cta;
+
+ ASSERT_TRUE(m_catalogue->getAllDiskSystems().empty());
+
+ const std::string name = "disk_system_name";
+ const std::string fileRegexp = "file_regexp";
+ const std::string freeSpaceQueryURL = "free_space_query_url";
+ const uint64_t refreshInterval = 32;
+ const uint64_t targetedFreeSpace = 64;
+ const uint64_t sleepTime = 15*60;
+ const std::string comment = "disk system comment";
+
+ m_catalogue->createDiskSystem(m_admin, name, fileRegexp,
+ freeSpaceQueryURL, refreshInterval, targetedFreeSpace, sleepTime, comment);
+
+ {
+ const auto diskSystemList = m_catalogue->getAllDiskSystems();
+ ASSERT_EQ(1, diskSystemList.size());
+
+ const auto &diskSystem = diskSystemList.front();
+ ASSERT_EQ(name, diskSystem.name);
+ ASSERT_EQ(fileRegexp, diskSystem.fileRegexp);
+ ASSERT_EQ(freeSpaceQueryURL, diskSystem.freeSpaceQueryURL);
+ ASSERT_EQ(refreshInterval, diskSystem.refreshInterval);
+ ASSERT_EQ(targetedFreeSpace, diskSystem.targetedFreeSpace);
+ ASSERT_EQ(comment, diskSystem.comment);
+
+ const auto creationLog = diskSystem.creationLog;
+ ASSERT_EQ(m_admin.username, creationLog.username);
+ ASSERT_EQ(m_admin.host, creationLog.host);
+
+ const auto lastModificationLog = diskSystem.lastModificationLog;
+ ASSERT_EQ(creationLog, lastModificationLog);
+ }
+
+ const uint64_t modifiedTargetedFreeSpace = 128;
+ m_catalogue->modifyDiskSystemTargetedFreeSpace(m_admin, name, modifiedTargetedFreeSpace);
+
+ {
+ const auto diskSystemList = m_catalogue->getAllDiskSystems();
+ ASSERT_EQ(1, diskSystemList.size());
+
+ const auto &diskSystem = diskSystemList.front();
+ ASSERT_EQ(name, diskSystem.name);
+ ASSERT_EQ(fileRegexp, diskSystem.fileRegexp);
+ ASSERT_EQ(freeSpaceQueryURL, diskSystem.freeSpaceQueryURL);
+ ASSERT_EQ(refreshInterval, diskSystem.refreshInterval);
+ ASSERT_EQ(modifiedTargetedFreeSpace, diskSystem.targetedFreeSpace);
+ ASSERT_EQ(comment, diskSystem.comment);
+
+ const auto creationLog = diskSystem.creationLog;
+ ASSERT_EQ(m_admin.username, creationLog.username);
+ ASSERT_EQ(m_admin.host, creationLog.host);
+ }
+}
+
+TEST_P(cta_catalogue_CatalogueTest, modifyDiskSystemTargetedFreeSpace_emptyStringDiskSystemName) {
+ using namespace cta;
+
+ ASSERT_TRUE(m_catalogue->getAllDiskSystems().empty());
+
+ const std::string diskSystemName = "";
+ const uint64_t modifiedTargetedFreeSpace = 128;
+ ASSERT_THROW(m_catalogue->modifyDiskSystemTargetedFreeSpace(m_admin, diskSystemName, modifiedTargetedFreeSpace),
+ catalogue::UserSpecifiedAnEmptyStringDiskSystemName);
+}
+
+TEST_P(cta_catalogue_CatalogueTest, modifyDiskSystemTargetedFreeSpace_nonExistentDiskSystemName) {
+ using namespace cta;
+
+ ASSERT_TRUE(m_catalogue->getAllDiskSystems().empty());
+
+ const std::string diskSystemName = "dummyDiskSystemName";
+ const uint64_t modifiedTargetedFreeSpace = 128;
+ ASSERT_THROW(m_catalogue->modifyDiskSystemTargetedFreeSpace(m_admin, diskSystemName, modifiedTargetedFreeSpace),
+ catalogue::UserSpecifiedANonExistentDiskSystem);
+}
+
+TEST_P(cta_catalogue_CatalogueTest, modifyDiskSystemTargetedFreeSpace_zeroTargetedFreeSpace) {
+ using namespace cta;
+
+ ASSERT_TRUE(m_catalogue->getAllDiskSystems().empty());
+
+ const std::string name = "disk_system_name";
+ const std::string fileRegexp = "file_regexp";
+ const std::string freeSpaceQueryURL = "free_space_query_url";
+ const uint64_t refreshInterval = 32;
+ const uint64_t targetedFreeSpace = 64;
+ const uint64_t sleepTime = 15*60;
+ const std::string comment = "disk system comment";
+
+ m_catalogue->createDiskSystem(m_admin, name, fileRegexp,
+ freeSpaceQueryURL, refreshInterval, targetedFreeSpace, sleepTime, comment);
+
+ {
+ const auto diskSystemList = m_catalogue->getAllDiskSystems();
+ ASSERT_EQ(1, diskSystemList.size());
+
+ const auto &diskSystem = diskSystemList.front();
+ ASSERT_EQ(name, diskSystem.name);
+ ASSERT_EQ(fileRegexp, diskSystem.fileRegexp);
+ ASSERT_EQ(freeSpaceQueryURL, diskSystem.freeSpaceQueryURL);
+ ASSERT_EQ(refreshInterval, diskSystem.refreshInterval);
+ ASSERT_EQ(targetedFreeSpace, diskSystem.targetedFreeSpace);
+ ASSERT_EQ(comment, diskSystem.comment);
+
+ const auto creationLog = diskSystem.creationLog;
+ ASSERT_EQ(m_admin.username, creationLog.username);
+ ASSERT_EQ(m_admin.host, creationLog.host);
+
+ const auto lastModificationLog = diskSystem.lastModificationLog;
+ ASSERT_EQ(creationLog, lastModificationLog);
+ }
+
+ const uint64_t modifiedTargetedFreeSpace = 0;
+ ASSERT_THROW(m_catalogue->modifyDiskSystemTargetedFreeSpace(m_admin, name, modifiedTargetedFreeSpace),
+ catalogue::UserSpecifiedAZeroTargetedFreeSpace);
+}
+
+TEST_P(cta_catalogue_CatalogueTest, modifyDiskSystemComment) {
+ using namespace cta;
+
+ ASSERT_TRUE(m_catalogue->getAllDiskSystems().empty());
+
+ const std::string name = "disk_system_name";
+ const std::string fileRegexp = "file_regexp";
+ const std::string freeSpaceQueryURL = "free_space_query_url";
+ const uint64_t refreshInterval = 32;
+ const uint64_t targetedFreeSpace = 64;
+ const uint64_t sleepTime = 15*60;
+ const std::string comment = "disk system comment";
+
+ m_catalogue->createDiskSystem(m_admin, name, fileRegexp,
+ freeSpaceQueryURL, refreshInterval, targetedFreeSpace, sleepTime, comment);
+
+ {
+ const auto diskSystemList = m_catalogue->getAllDiskSystems();
+ ASSERT_EQ(1, diskSystemList.size());
+
+ const auto &diskSystem = diskSystemList.front();
+ ASSERT_EQ(name, diskSystem.name);
+ ASSERT_EQ(fileRegexp, diskSystem.fileRegexp);
+ ASSERT_EQ(freeSpaceQueryURL, diskSystem.freeSpaceQueryURL);
+ ASSERT_EQ(refreshInterval, diskSystem.refreshInterval);
+ ASSERT_EQ(targetedFreeSpace, diskSystem.targetedFreeSpace);
+ ASSERT_EQ(comment, diskSystem.comment);
+
+ const auto creationLog = diskSystem.creationLog;
+ ASSERT_EQ(m_admin.username, creationLog.username);
+ ASSERT_EQ(m_admin.host, creationLog.host);
+
+ const auto lastModificationLog = diskSystem.lastModificationLog;
+ ASSERT_EQ(creationLog, lastModificationLog);
+ }
+
+ const std::string modifiedComment = "modified_comment";
+ m_catalogue->modifyDiskSystemComment(m_admin, name, modifiedComment);
+
+ {
+ const auto diskSystemList = m_catalogue->getAllDiskSystems();
+ ASSERT_EQ(1, diskSystemList.size());
+
+ const auto &diskSystem = diskSystemList.front();
+ ASSERT_EQ(name, diskSystem.name);
+ ASSERT_EQ(fileRegexp, diskSystem.fileRegexp);
+ ASSERT_EQ(freeSpaceQueryURL, diskSystem.freeSpaceQueryURL);
+ ASSERT_EQ(refreshInterval, diskSystem.refreshInterval);
+ ASSERT_EQ(targetedFreeSpace, diskSystem.targetedFreeSpace);
+ ASSERT_EQ(modifiedComment, diskSystem.comment);
+
+ const auto creationLog = diskSystem.creationLog;
+ ASSERT_EQ(m_admin.username, creationLog.username);
+ ASSERT_EQ(m_admin.host, creationLog.host);
+ }
+}
+
+TEST_P(cta_catalogue_CatalogueTest, modifyDiskSystemComment_emptyStringDiskSystemName) {
+ using namespace cta;
+
+ ASSERT_TRUE(m_catalogue->getAllDiskSystems().empty());
+
+ const std::string diskSystemName = "";
+ const std::string modifiedComment = "modified_comment";
+ ASSERT_THROW(m_catalogue->modifyDiskSystemComment(m_admin, diskSystemName, modifiedComment),
+ catalogue::UserSpecifiedAnEmptyStringDiskSystemName);
+}
+
+TEST_P(cta_catalogue_CatalogueTest, modifyDiskSystemComment_nonExistentDiskSystemName) {
+ using namespace cta;
+
+ ASSERT_TRUE(m_catalogue->getAllDiskSystems().empty());
+
+ const std::string diskSystemName = "dummyDiskSystemName";
+ const std::string modifiedComment = "modified_comment";
+ ASSERT_THROW(m_catalogue->modifyDiskSystemComment(m_admin, diskSystemName, modifiedComment),
+ catalogue::UserSpecifiedANonExistentDiskSystem);
+}
+
+TEST_P(cta_catalogue_CatalogueTest, modifyDiskSystemCommentL_emptyStringComment) {
+ using namespace cta;
+
+ ASSERT_TRUE(m_catalogue->getAllDiskSystems().empty());
+
+ const std::string name = "disk_system_name";
+ const std::string fileRegexp = "file_regexp";
+ const std::string freeSpaceQueryURL = "free_space_query_url";
+ const uint64_t refreshInterval = 32;
+ const uint64_t targetedFreeSpace = 64;
+ const uint64_t sleepTime = 15*60;
+ const std::string comment = "disk system comment";
+
+ m_catalogue->createDiskSystem(m_admin, name, fileRegexp,
+ freeSpaceQueryURL, refreshInterval, targetedFreeSpace, sleepTime, comment);
+
+ {
+ const auto diskSystemList = m_catalogue->getAllDiskSystems();
+ ASSERT_EQ(1, diskSystemList.size());
+
+ const auto &diskSystem = diskSystemList.front();
+ ASSERT_EQ(name, diskSystem.name);
+ ASSERT_EQ(fileRegexp, diskSystem.fileRegexp);
+ ASSERT_EQ(freeSpaceQueryURL, diskSystem.freeSpaceQueryURL);
+ ASSERT_EQ(refreshInterval, diskSystem.refreshInterval);
+ ASSERT_EQ(targetedFreeSpace, diskSystem.targetedFreeSpace);
+ ASSERT_EQ(comment, diskSystem.comment);
+
+ const auto creationLog = diskSystem.creationLog;
+ ASSERT_EQ(m_admin.username, creationLog.username);
+ ASSERT_EQ(m_admin.host, creationLog.host);
+
+ const auto lastModificationLog = diskSystem.lastModificationLog;
+ ASSERT_EQ(creationLog, lastModificationLog);
+ }
+
+ const std::string modifiedComment = "";
+ ASSERT_THROW(m_catalogue->modifyDiskSystemComment(m_admin, name, modifiedComment),
+ catalogue::UserSpecifiedAnEmptyStringComment);
+}
+
TEST_P(cta_catalogue_CatalogueTest, getNbFilesOnTape_no_tape_files) {
using namespace cta;
diff --git a/catalogue/DropSchemaCmd.cpp b/catalogue/DropSchemaCmd.cpp
index b154bbd33f08db29941082feb147872bbcb58304..40bf7fa116cf9703908ad7718043f1d99623cb11 100644
--- a/catalogue/DropSchemaCmd.cpp
+++ b/catalogue/DropSchemaCmd.cpp
@@ -149,7 +149,8 @@ void DropSchemaCmd::dropSqliteCatalogueSchema(rdbms::Conn &conn) {
"MOUNT_POLICY",
"ACTIVITIES_WEIGHTS",
"USAGESTATS",
- "EXPERIMENTS"
+ "EXPERIMENTS",
+ "DISK_SYSTEM"
};
dropDatabaseTables(conn, tablesToDrop);
} catch(exception::Exception &ex) {
@@ -180,7 +181,8 @@ void DropSchemaCmd::dropMysqlCatalogueSchema(rdbms::Conn &conn) {
"MOUNT_POLICY",
"ACTIVITIES_WEIGHTS",
"USAGESTATS",
- "EXPERIMENTS"
+ "EXPERIMENTS",
+ "DISK_SYSTEM"
};
dropDatabaseTables(conn, tablesToDrop);
@@ -248,7 +250,8 @@ void DropSchemaCmd::dropOracleCatalogueSchema(rdbms::Conn &conn) {
"MOUNT_POLICY",
"ACTIVITIES_WEIGHTS",
"USAGESTATS",
- "EXPERIMENTS"
+ "EXPERIMENTS",
+ "DISK_SYSTEM"
};
dropDatabaseTables(conn, tablesToDrop);
@@ -282,7 +285,8 @@ void DropSchemaCmd::dropPostgresCatalogueSchema(rdbms::Conn &conn) {
"MOUNT_POLICY",
"ACTIVITIES_WEIGHTS",
"USAGESTATS",
- "EXPERIMENTS"
+ "EXPERIMENTS",
+ "DISK_SYSTEM"
};
dropDatabaseTables(conn, tablesToDrop);
diff --git a/catalogue/DummyCatalogue.hpp b/catalogue/DummyCatalogue.hpp
index b314e32ed2e8d9b9f30ff534970a3cf8c9d5076e..4faf50c85f9a151d491128c8dfa6083006659c88 100644
--- a/catalogue/DummyCatalogue.hpp
+++ b/catalogue/DummyCatalogue.hpp
@@ -57,7 +57,16 @@ public:
void deleteTapePool(const std::string& name) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
void filesWrittenToTape(const std::set<TapeItemWrittenPointer>& event) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
std::list<common::dataStructures::ActivitiesFairShareWeights> getActivitiesFairShareWeights() const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
- std::list<common::dataStructures::AdminUser> getAdminUsers() const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
+ disk::DiskSystemList getAllDiskSystems() const override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
+ void createDiskSystem(const common::dataStructures::SecurityIdentity &admin, const std::string &name, const std::string &fileRegexp, const std::string &freeSpaceQueryURL, const uint64_t refreshInterval, const uint64_t targetedFreeSpace, const uint64_t sleepTime, const std::string &comment) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
+ void deleteDiskSystem(const std::string &name) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
+ void modifyDiskSystemFileRegexp(const common::dataStructures::SecurityIdentity &admin, const std::string &name, const std::string &fileRegexp) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented");}
+ void modifyDiskSystemFreeSpaceQueryURL(const common::dataStructures::SecurityIdentity &admin, const std::string &name, const std::string &freeSpaceQueryURL) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented");}
+ void modifyDiskSystemRefreshInterval(const common::dataStructures::SecurityIdentity &admin, const std::string &name, const uint64_t refreshInterval) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented");}
+ void modifyDiskSystemTargetedFreeSpace(const common::dataStructures::SecurityIdentity &admin, const std::string &name, const uint64_t targetedFreeSpace) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented");}
+ void modifyDiskSystemSleepTime(const common::dataStructures::SecurityIdentity& admin, const std::string& name, const uint64_t sleepTime) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented");}
+ void modifyDiskSystemComment(const common::dataStructures::SecurityIdentity &admin, const std::string &name, const std::string &comment) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented");}
+ std::list<common::dataStructures::AdminUser> getAdminUsers() const override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
common::dataStructures::ArchiveFile getArchiveFileById(const uint64_t id) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
ArchiveFileItor getArchiveFilesItor(const TapeFileSearchCriteria& searchCriteria) const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
std::list<common::dataStructures::ArchiveFile> getFilesForRepack(const std::string &vid, const uint64_t startFSeq, const uint64_t maxNbFiles) const override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
@@ -121,6 +130,7 @@ public:
void setTapeIsFromCastorInUnitTests(const std::string &vid) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
void setTapePoolEncryption(const common::dataStructures::SecurityIdentity& admin, const std::string& name, const bool encryptionValue) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
bool tapeExists(const std::string& vid) const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
+ bool diskSystemExists(const std::string& name) const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
void tapeLabelled(const std::string& vid, const std::string& drive) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
void tapeMountedForArchive(const std::string& vid, const std::string& drive) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
void tapeMountedForRetrieve(const std::string& vid, const std::string& drive) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
diff --git a/catalogue/RdbmsCatalogue.cpp b/catalogue/RdbmsCatalogue.cpp
index 663a2c33d6809c01991d3d8757d8ffce7457416e..ca64d317fcc403da9b999435abb4b08dfcf6562e 100644
--- a/catalogue/RdbmsCatalogue.cpp
+++ b/catalogue/RdbmsCatalogue.cpp
@@ -22,11 +22,18 @@
#include "catalogue/RdbmsCatalogueGetArchiveFilesForRepackItor.hpp"
#include "catalogue/retryOnLostConnection.hpp"
#include "catalogue/SqliteCatalogueSchema.hpp"
+#include "catalogue/UserSpecifiedANonExistentDiskSystem.hpp"
+#include "catalogue/UserSpecifiedANonEmptyDiskSystemAfterDelete.hpp"
#include "catalogue/UserSpecifiedANonEmptyLogicalLibrary.hpp"
#include "catalogue/UserSpecifiedANonEmptyTape.hpp"
#include "catalogue/UserSpecifiedANonExistentLogicalLibrary.hpp"
#include "catalogue/UserSpecifiedANonExistentTape.hpp"
#include "catalogue/UserSpecifiedAnEmptyStringComment.hpp"
+#include "catalogue/UserSpecifiedAnEmptyStringDiskSystemName.hpp"
+#include "catalogue/UserSpecifiedAnEmptyStringFileRegexp.hpp"
+#include "catalogue/UserSpecifiedAnEmptyStringFreeSpaceQueryURL.hpp"
+#include "catalogue/UserSpecifiedAZeroRefreshInterval.hpp"
+#include "catalogue/UserSpecifiedAZeroTargetedFreeSpace.hpp"
#include "catalogue/UserSpecifiedAnEmptyStringDiskInstanceName.hpp"
#include "catalogue/UserSpecifiedAnEmptyStringLogicalLibraryName.hpp"
#include "catalogue/UserSpecifiedAnEmptyStringMediaType.hpp"
@@ -2004,6 +2011,45 @@ bool RdbmsCatalogue::tapeExists(rdbms::Conn &conn, const std::string &vid) const
}
}
+//------------------------------------------------------------------------------
+// diskSystemExists
+//------------------------------------------------------------------------------
+bool RdbmsCatalogue::diskSystemExists(const std::string &name) const {
+ try {
+ auto conn = m_connPool.getConn();
+ return diskSystemExists(conn, name);
+ } catch(exception::UserError &) {
+ throw;
+ } catch(exception::Exception &ex) {
+ ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str());
+ throw;
+ }
+}
+
+//------------------------------------------------------------------------------
+// diskSystemExists
+//------------------------------------------------------------------------------
+bool RdbmsCatalogue::diskSystemExists(rdbms::Conn &conn, const std::string &name) const {
+ try {
+ const char *const sql =
+ "SELECT "
+ "DISK_SYSTEM_NAME AS DISK_SYSTEM_NAME "
+ "FROM "
+ "DISK_SYSTEM "
+ "WHERE "
+ "DISK_SYSTEM_NAME = :DISK_SYSTEM_NAME";
+ auto stmt = conn.createStmt(sql);
+ stmt.bindString(":DISK_SYSTEM_NAME", name);
+ auto rset = stmt.executeQuery();
+ return rset.next();
+ } catch(exception::UserError &) {
+ throw;
+ } catch(exception::Exception &ex) {
+ ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str());
+ throw;
+ }
+}
+
//------------------------------------------------------------------------------
// deleteTape
//------------------------------------------------------------------------------
@@ -4637,6 +4683,450 @@ std::list<common::dataStructures::ActivitiesFairShareWeights> RdbmsCatalogue::ge
}
}
+//------------------------------------------------------------------------------
+// getAllDiskSystems
+//------------------------------------------------------------------------------
+disk::DiskSystemList RdbmsCatalogue::getAllDiskSystems() const {
+ try {
+ disk::DiskSystemList diskSystemList;
+ std::string sql =
+ "SELECT "
+ "DISK_SYSTEM.DISK_SYSTEM_NAME AS DISK_SYSTEM_NAME,"
+ "DISK_SYSTEM.FILE_REGEXP AS FILE_REGEXP,"
+ "DISK_SYSTEM.FREE_SPACE_QUERY_URL AS FREE_SPACE_QUERY_URL,"
+ "DISK_SYSTEM.REFRESH_INTERVAL AS REFRESH_INTERVAL,"
+ "DISK_SYSTEM.TARGETED_FREE_SPACE AS TARGETED_FREE_SPACE,"
+ "DISK_SYSTEM.SLEEP_TIME AS SLEEP_TIME,"
+
+ "DISK_SYSTEM.USER_COMMENT AS USER_COMMENT,"
+
+ "DISK_SYSTEM.CREATION_LOG_USER_NAME AS CREATION_LOG_USER_NAME,"
+ "DISK_SYSTEM.CREATION_LOG_HOST_NAME AS CREATION_LOG_HOST_NAME,"
+ "DISK_SYSTEM.CREATION_LOG_TIME AS CREATION_LOG_TIME,"
+
+ "DISK_SYSTEM.LAST_UPDATE_USER_NAME AS LAST_UPDATE_USER_NAME,"
+ "DISK_SYSTEM.LAST_UPDATE_HOST_NAME AS LAST_UPDATE_HOST_NAME,"
+ "DISK_SYSTEM.LAST_UPDATE_TIME AS LAST_UPDATE_TIME "
+ "FROM "
+ "DISK_SYSTEM";
+
+ auto conn = m_connPool.getConn();
+ auto stmt = conn.createStmt(sql);
+
+ auto rset = stmt.executeQuery();
+ while (rset.next()) {
+ disk::DiskSystem diskSystem;
+ diskSystem.name = rset.columnString("DISK_SYSTEM_NAME");
+ diskSystem.fileRegexp = rset.columnString("FILE_REGEXP");
+ diskSystem.freeSpaceQueryURL = rset.columnString("FREE_SPACE_QUERY_URL");
+ diskSystem.refreshInterval = rset.columnUint64("REFRESH_INTERVAL");
+ diskSystem.targetedFreeSpace = rset.columnUint64("TARGETED_FREE_SPACE");
+ diskSystem.sleepTime = rset.columnUint64("SLEEP_TIME");
+ diskSystem.comment = rset.columnString("USER_COMMENT");
+ diskSystem.creationLog.username = rset.columnString("CREATION_LOG_USER_NAME");
+ diskSystem.creationLog.host = rset.columnString("CREATION_LOG_HOST_NAME");
+ diskSystem.creationLog.time = rset.columnUint64("CREATION_LOG_TIME");
+ diskSystem.lastModificationLog.username = rset.columnString("LAST_UPDATE_USER_NAME");
+ diskSystem.lastModificationLog.host = rset.columnString("LAST_UPDATE_HOST_NAME");
+ diskSystem.lastModificationLog.time = rset.columnUint64("LAST_UPDATE_TIME");
+ diskSystemList.push_back(diskSystem);
+ }
+ return diskSystemList;
+ } catch(exception::UserError &) {
+ throw;
+ } catch(exception::Exception &ex) {
+ ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str());
+ throw;
+ }
+}
+
+//------------------------------------------------------------------------------
+// createDiskSystem
+//------------------------------------------------------------------------------
+void RdbmsCatalogue::createDiskSystem(
+ const common::dataStructures::SecurityIdentity &admin,
+ const std::string &name,
+ const std::string &fileRegexp,
+ const std::string &freeSpaceQueryURL,
+ const uint64_t refreshInterval,
+ const uint64_t targetedFreeSpace,
+ const uint64_t sleepTime,
+ const std::string &comment) {
+ try {
+ if(name.empty()) {
+ throw UserSpecifiedAnEmptyStringDiskSystemName("Cannot create disk system because the name is an empty string");
+ }
+ if(fileRegexp.empty()) {
+ throw UserSpecifiedAnEmptyStringFileRegexp("Cannot create disk system because the file regexp is an empty string");
+ }
+ if(freeSpaceQueryURL.empty()) {
+ throw UserSpecifiedAnEmptyStringFreeSpaceQueryURL("Cannot create disk system because the free space query URL is an empty string");
+ }
+ if(0 == refreshInterval) {
+ throw UserSpecifiedAZeroRefreshInterval("Cannot create disk system because the refresh interval is zero");
+ }
+ if(0 == targetedFreeSpace) {
+ throw UserSpecifiedAZeroTargetedFreeSpace("Cannot create disk system because the targeted free space is zero");
+ }
+ if (0 == sleepTime) {
+ throw UserSpecifiedAZeroSleepTime("Cannot create disk system because the sleep time is zero");
+ }
+ if(comment.empty()) {
+ throw UserSpecifiedAnEmptyStringComment("Cannot create disk system because the comment is an empty string");
+ }
+
+ auto conn = m_connPool.getConn();
+ if(diskSystemExists(conn, name)) {
+ throw exception::UserError(std::string("Cannot create disk system ") + name +
+ " because a disk system with the same name identifier already exists");
+ }
+
+ const time_t now = time(nullptr);
+ const char *const sql =
+ "INSERT INTO DISK_SYSTEM("
+ "DISK_SYSTEM_NAME,"
+ "FILE_REGEXP,"
+ "FREE_SPACE_QUERY_URL,"
+ "REFRESH_INTERVAL,"
+ "TARGETED_FREE_SPACE,"
+ "SLEEP_TIME,"
+
+ "USER_COMMENT,"
+
+ "CREATION_LOG_USER_NAME,"
+ "CREATION_LOG_HOST_NAME,"
+ "CREATION_LOG_TIME,"
+
+ "LAST_UPDATE_USER_NAME,"
+ "LAST_UPDATE_HOST_NAME,"
+ "LAST_UPDATE_TIME)"
+ "VALUES("
+ ":DISK_SYSTEM_NAME,"
+ ":FILE_REGEXP,"
+ ":FREE_SPACE_QUERY_URL,"
+ ":REFRESH_INTERVAL,"
+ ":TARGETED_FREE_SPACE,"
+ ":SLEEP_TIME,"
+
+ ":USER_COMMENT,"
+
+ ":CREATION_LOG_USER_NAME,"
+ ":CREATION_LOG_HOST_NAME,"
+ ":CREATION_LOG_TIME,"
+
+ ":LAST_UPDATE_USER_NAME,"
+ ":LAST_UPDATE_HOST_NAME,"
+ ":LAST_UPDATE_TIME)";
+ auto stmt = conn.createStmt(sql);
+
+ stmt.bindString(":DISK_SYSTEM_NAME", name);
+ stmt.bindString(":FILE_REGEXP", fileRegexp);
+ stmt.bindString(":FREE_SPACE_QUERY_URL", freeSpaceQueryURL);
+ stmt.bindUint64(":REFRESH_INTERVAL", refreshInterval);
+ stmt.bindUint64(":TARGETED_FREE_SPACE", targetedFreeSpace);
+ stmt.bindUint64(":SLEEP_TIME", sleepTime);
+
+ stmt.bindString(":USER_COMMENT", comment);
+
+ stmt.bindString(":CREATION_LOG_USER_NAME", admin.username);
+ stmt.bindString(":CREATION_LOG_HOST_NAME", admin.host);
+ stmt.bindUint64(":CREATION_LOG_TIME", now);
+
+ stmt.bindString(":LAST_UPDATE_USER_NAME", admin.username);
+ stmt.bindString(":LAST_UPDATE_HOST_NAME", admin.host);
+ stmt.bindUint64(":LAST_UPDATE_TIME", now);
+
+ stmt.executeNonQuery();
+ } catch(exception::UserError &) {
+ throw;
+ } catch(exception::Exception &ex) {
+ ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str());
+ throw;
+ }
+}
+
+//------------------------------------------------------------------------------
+// deleteDiskSystem
+//------------------------------------------------------------------------------
+void RdbmsCatalogue::deleteDiskSystem(const std::string &name) {
+ try {
+ const char *const delete_sql =
+ "DELETE "
+ "FROM "
+ "DISK_SYSTEM "
+ "WHERE "
+ "DISK_SYSTEM_NAME = :DISK_SYSTEM_NAME";
+ auto conn = m_connPool.getConn();
+ auto stmt = conn.createStmt(delete_sql);
+ stmt.bindString(":DISK_SYSTEM_NAME", name);
+ stmt.executeNonQuery();
+
+ // The delete statement will effect no rows and will not raise an error if
+ // either the tape does not exist or if it still has tape files
+ if(0 == stmt.getNbAffectedRows()) {
+ if(diskSystemExists(conn, name)) {
+ throw UserSpecifiedANonEmptyDiskSystemAfterDelete(std::string("Cannot delete disk system ") + name + " for unknown reason");
+ } else {
+ throw UserSpecifiedANonExistentDiskSystem(std::string("Cannot delete disk system ") + name + " because it does not exist");
+ }
+ }
+ } catch(exception::UserError &) {
+ throw;
+ } catch(exception::Exception &ex) {
+ ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str());
+ throw;
+ }
+}
+
+void RdbmsCatalogue::modifyDiskSystemFileRegexp(const common::dataStructures::SecurityIdentity &admin,
+ const std::string &name, const std::string &fileRegexp) {
+ try {
+ if(name.empty()) {
+ throw UserSpecifiedAnEmptyStringDiskSystemName("Cannot modify disk system"
+ " because the disk system name is an empty string");
+ }
+ if(fileRegexp.empty()) {
+ throw UserSpecifiedAnEmptyStringFileRegexp("Cannot modify disk system "
+ "because the new fileRegexp is an empty string");
+ }
+
+ const time_t now = time(nullptr);
+ const char *const sql =
+ "UPDATE DISK_SYSTEM SET "
+ "FILE_REGEXP = :FILE_REGEXP,"
+ "LAST_UPDATE_USER_NAME = :LAST_UPDATE_USER_NAME,"
+ "LAST_UPDATE_HOST_NAME = :LAST_UPDATE_HOST_NAME,"
+ "LAST_UPDATE_TIME = :LAST_UPDATE_TIME "
+ "WHERE "
+ "DISK_SYSTEM_NAME = :DISK_SYSTEM_NAME";
+ auto conn = m_connPool.getConn();
+ auto stmt = conn.createStmt(sql);
+ stmt.bindString(":FILE_REGEXP", fileRegexp);
+ stmt.bindString(":LAST_UPDATE_USER_NAME", admin.username);
+ stmt.bindString(":LAST_UPDATE_HOST_NAME", admin.host);
+ stmt.bindUint64(":LAST_UPDATE_TIME", now);
+ stmt.bindString(":DISK_SYSTEM_NAME", name);
+ stmt.executeNonQuery();
+
+ if(0 == stmt.getNbAffectedRows()) {
+ throw UserSpecifiedANonExistentDiskSystem(std::string("Cannot modify disk system ") + name + " because it does not exist");
+ }
+ } catch(exception::UserError &) {
+ throw;
+ } catch(exception::Exception &ex) {
+ ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str());
+ throw;
+ }
+}
+
+void RdbmsCatalogue::modifyDiskSystemFreeSpaceQueryURL(const common::dataStructures::SecurityIdentity &admin,
+ const std::string &name, const std::string &freeSpaceQueryURL) {
+ try {
+ if(name.empty()) {
+ throw UserSpecifiedAnEmptyStringDiskSystemName("Cannot modify disk system"
+ " because the disk system name is an empty string");
+ }
+ if(freeSpaceQueryURL.empty()) {
+ throw UserSpecifiedAnEmptyStringFreeSpaceQueryURL("Cannot modify disk system "
+ "because the new freeSpaceQueryURL is an empty string");
+ }
+
+ const time_t now = time(nullptr);
+ const char *const sql =
+ "UPDATE DISK_SYSTEM SET "
+ "FREE_SPACE_QUERY_URL = :FREE_SPACE_QUERY_URL,"
+ "LAST_UPDATE_USER_NAME = :LAST_UPDATE_USER_NAME,"
+ "LAST_UPDATE_HOST_NAME = :LAST_UPDATE_HOST_NAME,"
+ "LAST_UPDATE_TIME = :LAST_UPDATE_TIME "
+ "WHERE "
+ "DISK_SYSTEM_NAME = :DISK_SYSTEM_NAME";
+ auto conn = m_connPool.getConn();
+ auto stmt = conn.createStmt(sql);
+ stmt.bindString(":FREE_SPACE_QUERY_URL", freeSpaceQueryURL);
+ stmt.bindString(":LAST_UPDATE_USER_NAME", admin.username);
+ stmt.bindString(":LAST_UPDATE_HOST_NAME", admin.host);
+ stmt.bindUint64(":LAST_UPDATE_TIME", now);
+ stmt.bindString(":DISK_SYSTEM_NAME", name);
+ stmt.executeNonQuery();
+
+ if(0 == stmt.getNbAffectedRows()) {
+ throw UserSpecifiedANonExistentDiskSystem(std::string("Cannot modify disk system ") + name + " because it does not exist");
+ }
+ } catch(exception::UserError &) {
+ throw;
+ } catch(exception::Exception &ex) {
+ ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str());
+ throw;
+ }
+}
+
+void RdbmsCatalogue::modifyDiskSystemRefreshInterval(const common::dataStructures::SecurityIdentity &admin,
+ const std::string &name, const uint64_t refreshInterval) {
+ try {
+ if(name.empty()) {
+ throw UserSpecifiedAnEmptyStringDiskSystemName("Cannot modify disk system"
+ " because the disk system name is an empty string");
+ }
+ if(0 == refreshInterval) {
+ throw UserSpecifiedAZeroRefreshInterval("Cannot modify disk system "
+ "because the new refresh interval has zero value");
+ }
+
+ const time_t now = time(nullptr);
+ const char *const sql =
+ "UPDATE DISK_SYSTEM SET "
+ "REFRESH_INTERVAL = :REFRESH_INTERVAL,"
+ "LAST_UPDATE_USER_NAME = :LAST_UPDATE_USER_NAME,"
+ "LAST_UPDATE_HOST_NAME = :LAST_UPDATE_HOST_NAME,"
+ "LAST_UPDATE_TIME = :LAST_UPDATE_TIME "
+ "WHERE "
+ "DISK_SYSTEM_NAME = :DISK_SYSTEM_NAME";
+ auto conn = m_connPool.getConn();
+ auto stmt = conn.createStmt(sql);
+ stmt.bindUint64(":REFRESH_INTERVAL", refreshInterval);
+ stmt.bindString(":LAST_UPDATE_USER_NAME", admin.username);
+ stmt.bindString(":LAST_UPDATE_HOST_NAME", admin.host);
+ stmt.bindUint64(":LAST_UPDATE_TIME", now);
+ stmt.bindString(":DISK_SYSTEM_NAME", name);
+ stmt.executeNonQuery();
+
+ if(0 == stmt.getNbAffectedRows()) {
+ throw UserSpecifiedANonExistentDiskSystem(std::string("Cannot modify disk system ") + name + " because it does not exist");
+ }
+ } catch(exception::UserError &) {
+ throw;
+ } catch(exception::Exception &ex) {
+ ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str());
+ throw;
+ }
+}
+
+void RdbmsCatalogue::modifyDiskSystemTargetedFreeSpace(const common::dataStructures::SecurityIdentity &admin,
+ const std::string &name, const uint64_t targetedFreeSpace) {
+ try {
+ if(name.empty()) {
+ throw UserSpecifiedAnEmptyStringDiskSystemName("Cannot modify disk system"
+ " because the disk system name is an empty string");
+ }
+ if(0 == targetedFreeSpace) {
+ throw UserSpecifiedAZeroTargetedFreeSpace("Cannot modify disk system "
+ "because the new targeted free space has zero value");
+ }
+
+ const time_t now = time(nullptr);
+ const char *const sql =
+ "UPDATE DISK_SYSTEM SET "
+ "TARGETED_FREE_SPACE = :TARGETED_FREE_SPACE,"
+ "LAST_UPDATE_USER_NAME = :LAST_UPDATE_USER_NAME,"
+ "LAST_UPDATE_HOST_NAME = :LAST_UPDATE_HOST_NAME,"
+ "LAST_UPDATE_TIME = :LAST_UPDATE_TIME "
+ "WHERE "
+ "DISK_SYSTEM_NAME = :DISK_SYSTEM_NAME";
+ auto conn = m_connPool.getConn();
+ auto stmt = conn.createStmt(sql);
+ stmt.bindUint64(":TARGETED_FREE_SPACE", targetedFreeSpace);
+ stmt.bindString(":LAST_UPDATE_USER_NAME", admin.username);
+ stmt.bindString(":LAST_UPDATE_HOST_NAME", admin.host);
+ stmt.bindUint64(":LAST_UPDATE_TIME", now);
+ stmt.bindString(":DISK_SYSTEM_NAME", name);
+ stmt.executeNonQuery();
+
+ if(0 == stmt.getNbAffectedRows()) {
+ throw UserSpecifiedANonExistentDiskSystem(std::string("Cannot modify disk system ") + name + " because it does not exist");
+ }
+ } catch(exception::UserError &) {
+ throw;
+ } catch(exception::Exception &ex) {
+ ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str());
+ throw;
+ }
+}
+
+void RdbmsCatalogue::modifyDiskSystemComment(const common::dataStructures::SecurityIdentity &admin,
+ const std::string &name, const std::string &comment) {
+ try {
+ if(name.empty()) {
+ throw UserSpecifiedAnEmptyStringDiskSystemName("Cannot modify disk system"
+ " because the disk system name is an empty string");
+ }
+ if(comment.empty()) {
+ throw UserSpecifiedAnEmptyStringComment("Cannot modify disk system "
+ "because the new comment is an empty string");
+ }
+
+ const time_t now = time(nullptr);
+ const char *const sql =
+ "UPDATE DISK_SYSTEM SET "
+ "USER_COMMENT = :USER_COMMENT,"
+ "LAST_UPDATE_USER_NAME = :LAST_UPDATE_USER_NAME,"
+ "LAST_UPDATE_HOST_NAME = :LAST_UPDATE_HOST_NAME,"
+ "LAST_UPDATE_TIME = :LAST_UPDATE_TIME "
+ "WHERE "
+ "DISK_SYSTEM_NAME = :DISK_SYSTEM_NAME";
+ auto conn = m_connPool.getConn();
+ auto stmt = conn.createStmt(sql);
+ stmt.bindString(":USER_COMMENT", comment);
+ stmt.bindString(":LAST_UPDATE_USER_NAME", admin.username);
+ stmt.bindString(":LAST_UPDATE_HOST_NAME", admin.host);
+ stmt.bindUint64(":LAST_UPDATE_TIME", now);
+ stmt.bindString(":DISK_SYSTEM_NAME", name);
+ stmt.executeNonQuery();
+
+ if(0 == stmt.getNbAffectedRows()) {
+ throw UserSpecifiedANonExistentDiskSystem(std::string("Cannot modify disk system ") + name + " because it does not exist");
+ }
+ } catch(exception::UserError &) {
+ throw;
+ } catch(exception::Exception &ex) {
+ ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str());
+ throw;
+ }
+}
+
+//------------------------------------------------------------------------------
+// modifyDiskSystemSleepTime
+//------------------------------------------------------------------------------
+void RdbmsCatalogue::modifyDiskSystemSleepTime(const common::dataStructures::SecurityIdentity& admin, const std::string& name,
+ const uint64_t sleepTime) {
+ try {
+ if(name.empty()) {
+ throw UserSpecifiedAnEmptyStringDiskSystemName("Cannot modify disk system"
+ " because the disk system name is an empty string");
+ }
+ if(sleepTime == 0) {
+ throw UserSpecifiedAZeroSleepTime("Cannot modify disk system "
+ "because the new sleep time is zero");
+ }
+
+ const time_t now = time(nullptr);
+ const char *const sql =
+ "UPDATE DISK_SYSTEM SET "
+ "SLEEP_TIME = :SLEEP_TIME,"
+ "LAST_UPDATE_USER_NAME = :LAST_UPDATE_USER_NAME,"
+ "LAST_UPDATE_HOST_NAME = :LAST_UPDATE_HOST_NAME,"
+ "LAST_UPDATE_TIME = :LAST_UPDATE_TIME "
+ "WHERE "
+ "DISK_SYSTEM_NAME = :DISK_SYSTEM_NAME";
+ auto conn = m_connPool.getConn();
+ auto stmt = conn.createStmt(sql);
+ stmt.bindUint64(":SLEEP_TIME", sleepTime);
+ stmt.bindString(":LAST_UPDATE_USER_NAME", admin.username);
+ stmt.bindString(":LAST_UPDATE_HOST_NAME", admin.host);
+ stmt.bindUint64(":LAST_UPDATE_TIME", now);
+ stmt.bindString(":DISK_SYSTEM_NAME", name);
+ stmt.executeNonQuery();
+
+ if(0 == stmt.getNbAffectedRows()) {
+ throw UserSpecifiedANonExistentDiskSystem(std::string("Cannot modify disk system ") + name + " because it does not exist");
+ }
+ } catch(exception::UserError &) {
+ throw;
+ } catch(exception::Exception &ex) {
+ ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str());
+ throw;
+ }
+}
+
//------------------------------------------------------------------------------
// insertArchiveFile
//------------------------------------------------------------------------------
diff --git a/catalogue/RdbmsCatalogue.hpp b/catalogue/RdbmsCatalogue.hpp
index 4e5a9515b5624061341ec897a6d98e8d14f87445..8de37183d37975c593b9b26a5a86e9daaad5ebdb 100644
--- a/catalogue/RdbmsCatalogue.hpp
+++ b/catalogue/RdbmsCatalogue.hpp
@@ -561,6 +561,60 @@ public:
void deleteActivitiesFairShareWeight(const common::dataStructures::SecurityIdentity &admin, const std::string & diskInstanceName, const std::string & activity) override;
std::list<common::dataStructures::ActivitiesFairShareWeights> getActivitiesFairShareWeights() const override;
+ /**
+ * Returns all the disk systems within the CTA catalogue.
+ *
+ * @return The disk systems.
+ * requester group.
+ */
+ disk::DiskSystemList getAllDiskSystems() const override;
+
+ /**
+ * Creates a disk system.
+ *
+ * @param admin The administrator.
+ * @param name The name of the disk system.
+ * @param fileRegexp The regular expression allowing matching destination URLs
+ * for this disk system.
+ * @param freeSpaceQueryURL The query URL that describes a method to query the
+ * free space from the disk system.
+ * @param refreshInterval The refresh interval (seconds) defining how long do
+ * we use a free space value.
+ * @param targetedFreeSpace The targeted free space (margin) based on the free
+ * space update latency (inherent to the file system and induced by the refresh
+ * interval), and the expected external bandwidth from sources external to CTA.
+ * @param comment Comment.
+ */
+ void createDiskSystem(
+ const common::dataStructures::SecurityIdentity &admin,
+ const std::string &name,
+ const std::string &fileRegexp,
+ const std::string &freeSpaceQueryURL,
+ const uint64_t refreshInterval,
+ const uint64_t targetedFreeSpace,
+ const uint64_t sleepTime,
+ const std::string &comment) override;
+
+ /**
+ * Deletes a disk system.
+ *
+ * @param name The name of the disk system.
+ */
+ void deleteDiskSystem(const std::string &name) override;
+
+ void modifyDiskSystemFileRegexp(const common::dataStructures::SecurityIdentity &admin,
+ const std::string &name, const std::string &fileRegexp) override;
+ void modifyDiskSystemFreeSpaceQueryURL(const common::dataStructures::SecurityIdentity &admin,
+ const std::string &name, const std::string &freeSpaceQueryURL) override;
+ void modifyDiskSystemRefreshInterval(const common::dataStructures::SecurityIdentity &admin,
+ const std::string &name, const uint64_t refreshInterval) override;
+ void modifyDiskSystemTargetedFreeSpace(const common::dataStructures::SecurityIdentity &admin,
+ const std::string &name, const uint64_t targetedFreeSpace) override;
+ void modifyDiskSystemComment(const common::dataStructures::SecurityIdentity &admin,
+ const std::string &name, const std::string &comment) override;
+ void modifyDiskSystemSleepTime(const common::dataStructures::SecurityIdentity& admin,
+ const std::string& name, const uint64_t sleepTime) override;
+
/**
* Throws a UserError exception if the specified searchCriteria is not valid
@@ -806,6 +860,23 @@ protected:
* @return True if the tape exists.
*/
bool tapeExists(rdbms::Conn &conn, const std::string &vid) const;
+
+ /**
+ * Returns true if the specified disk system exists.
+ *
+ * @param name The name identifier of the disk system.
+ * @return True if the tape exists.
+ */
+ bool diskSystemExists(const std::string &name) const override;
+
+ /**
+ * Returns true if the specified disk system exists.
+ *
+ * @param conn The database connection.
+ * @param name The name identifier of the disk system.
+ * @return True if the disk system exists.
+ */
+ bool diskSystemExists(rdbms::Conn &conn, const std::string &name) const;
/**
* Returns the list of tapes that meet the specified search criteria.
diff --git a/objectstore/OwnerIdentitySerDeser.hpp b/catalogue/UserSpecifiedANonEmptyDiskSystemAfterDelete.cpp
similarity index 56%
rename from objectstore/OwnerIdentitySerDeser.hpp
rename to catalogue/UserSpecifiedANonEmptyDiskSystemAfterDelete.cpp
index e54d396479b1124cdb4cadb197a1f925554ad62a..8e0ce64760eee2cf19aaa98a0d038208a4a796cc 100644
--- a/objectstore/OwnerIdentitySerDeser.hpp
+++ b/catalogue/UserSpecifiedANonEmptyDiskSystemAfterDelete.cpp
@@ -1,6 +1,6 @@
-/**
+/*
* The CERN Tape Archive (CTA) project
- * Copyright (C) 2019 CERN
+ * Copyright (C) 2015 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@@ -16,24 +16,17 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
-#pragma once
+#include "UserSpecifiedANonEmptyDiskSystemAfterDelete.hpp"
-#include <string>
+namespace cta {
+namespace catalogue {
-#include "common/dataStructures/OwnerIdentity.hpp"
-#include "objectstore/cta.pb.h"
+//------------------------------------------------------------------------------
+// constructor
+//------------------------------------------------------------------------------
+UserSpecifiedANonEmptyDiskSystemAfterDelete::UserSpecifiedANonEmptyDiskSystemAfterDelete(const std::string &context, const bool embedBacktrace):
+ UserError(context, embedBacktrace) {
+}
-namespace cta { namespace objectstore {
-
-struct OwnerIdentitySerDeser: public cta::common::dataStructures::OwnerIdentity {
-
- void serialize(cta::objectstore::serializers::OwnerIdentity &user) const {
- user.set_uid(uid);
- user.set_gid(gid);
- }
-
- void deserialize(const cta::objectstore::serializers::OwnerIdentity &user) :
- uid(user.uid()), gid(user.gid()) {}
-};
-
-}}
+} // namespace catalogue
+} // namespace cta
diff --git a/catalogue/UserSpecifiedANonEmptyDiskSystemAfterDelete.hpp b/catalogue/UserSpecifiedANonEmptyDiskSystemAfterDelete.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..fd679e7fb25d6d0e857e76cce59ba4e18ecabc35
--- /dev/null
+++ b/catalogue/UserSpecifiedANonEmptyDiskSystemAfterDelete.hpp
@@ -0,0 +1,45 @@
+/*
+ * The CERN Tape Archive (CTA) project
+ * Copyright (C) 2015 CERN
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include "common/exception/UserError.hpp"
+
+namespace cta {
+namespace catalogue {
+
+/**
+ * User error thrown when a disk system was not deleted for unknown reason.
+ */
+class UserSpecifiedANonEmptyDiskSystemAfterDelete: public exception::UserError {
+public:
+
+ /**
+ * Constructor.
+ *
+ * @param context optional context string added to the message
+ * at initialisation time.
+ * @param embedBacktrace whether to embed a backtrace of where the
+ * exception was throw in the message
+ */
+ UserSpecifiedANonEmptyDiskSystemAfterDelete(const std::string &context = "", const bool embedBacktrace = true);
+
+}; // class UserSpecifiedANonEmptyDiskSystemAfterDelete
+
+} // namespace catalogue
+} // namespace cta
diff --git a/objectstore/RequesterIdentitySerDeser.hpp b/catalogue/UserSpecifiedANonExistentDiskSystem.cpp
similarity index 58%
rename from objectstore/RequesterIdentitySerDeser.hpp
rename to catalogue/UserSpecifiedANonExistentDiskSystem.cpp
index c4ba50f29e1118b235bb1a4915ac0b52e5a66564..f5ad9a9348673ab53a76adebe062dfb80deb56c7 100644
--- a/objectstore/RequesterIdentitySerDeser.hpp
+++ b/catalogue/UserSpecifiedANonExistentDiskSystem.cpp
@@ -16,24 +16,17 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
-#pragma once
+#include "UserSpecifiedANonExistentDiskSystem.hpp"
-#include <string>
+namespace cta {
+namespace catalogue {
-#include "common/dataStructures/RequesterIdentity.hpp"
-#include "objectstore/cta.pb.h"
+//------------------------------------------------------------------------------
+// constructor
+//------------------------------------------------------------------------------
+UserSpecifiedANonExistentDiskSystem::UserSpecifiedANonExistentDiskSystem(const std::string &context, const bool embedBacktrace):
+ UserError(context, embedBacktrace) {
+}
-namespace cta { namespace objectstore {
-
-struct RequesterIdentitySerDeser: public cta::common::dataStructures::RequesterIdentity {
-
- void serialize (cta::objectstore::serializers::RequesterIdentity & user) const {
- user.set_name(name);
- user.set_group(group);
- }
-
- void deserialize (const cta::objectstore::serializers::RequesterIdentity & user) :
- name(user.name()), group(user.group()) {}
-};
-
-}}
+} // namespace catalogue
+} // namespace cta
diff --git a/catalogue/UserSpecifiedANonExistentDiskSystem.hpp b/catalogue/UserSpecifiedANonExistentDiskSystem.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..05dd1da503d837c0431cc8175c1c057fd599af61
--- /dev/null
+++ b/catalogue/UserSpecifiedANonExistentDiskSystem.hpp
@@ -0,0 +1,45 @@
+/*
+ * The CERN Tape Archive (CTA) project
+ * Copyright (C) 2015 CERN
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include "common/exception/UserError.hpp"
+
+namespace cta {
+namespace catalogue {
+
+/**
+ * User error thrown when a disk system specified does not exists.
+ */
+class UserSpecifiedANonExistentDiskSystem: public exception::UserError {
+public:
+
+ /**
+ * Constructor.
+ *
+ * @param context optional context string added to the message
+ * at initialisation time.
+ * @param embedBacktrace whether to embed a backtrace of where the
+ * exception was throw in the message
+ */
+ UserSpecifiedANonExistentDiskSystem(const std::string &context = "", const bool embedBacktrace = true);
+
+}; // class UserSpecifiedANonExistentDiskSystem
+
+} // namespace catalogue
+} // namespace cta
diff --git a/catalogue/UserSpecifiedAZeroRefreshInterval.cpp b/catalogue/UserSpecifiedAZeroRefreshInterval.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..689ee12e2825755f55f32d9e84ff2bd3f0c53a9a
--- /dev/null
+++ b/catalogue/UserSpecifiedAZeroRefreshInterval.cpp
@@ -0,0 +1,39 @@
+/*
+ * The CERN Tape Archive (CTA) project
+ * Copyright (C) 2015 CERN
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "catalogue/UserSpecifiedAZeroRefreshInterval.hpp"
+
+namespace cta {
+namespace catalogue {
+
+
+//------------------------------------------------------------------------------
+// constructor
+//------------------------------------------------------------------------------
+UserSpecifiedAZeroRefreshInterval::UserSpecifiedAZeroRefreshInterval(const std::string &context, const bool embedBacktrace):
+ cta::exception::UserError(context, embedBacktrace) {
+}
+
+//------------------------------------------------------------------------------
+// destructor
+//------------------------------------------------------------------------------
+UserSpecifiedAZeroRefreshInterval::~UserSpecifiedAZeroRefreshInterval() {
+}
+
+} // namespace catalogue
+} // namespace cta
diff --git a/catalogue/UserSpecifiedAZeroRefreshInterval.hpp b/catalogue/UserSpecifiedAZeroRefreshInterval.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..08680ce1602466c68001d30897b0c55f4175299e
--- /dev/null
+++ b/catalogue/UserSpecifiedAZeroRefreshInterval.hpp
@@ -0,0 +1,48 @@
+/*
+ * The CERN Tape Archive (CTA) project
+ * Copyright (C) 2015 CERN
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include "common/exception/UserError.hpp"
+
+namespace cta {
+namespace catalogue {
+
+/**
+ * User specified a zero refresh interval when this is not permitted.
+ */
+class UserSpecifiedAZeroRefreshInterval: public exception::UserError {
+public:
+ /**
+ * Constructor.
+ *
+ * @param context optional context string added to the message
+ * at initialisation time.
+ * @param embedBacktrace whether to embed a backtrace of where the
+ * exception was throw in the message
+ */
+ UserSpecifiedAZeroRefreshInterval(const std::string &context = "", const bool embedBacktrace = true);
+
+ /**
+ * Destructor.
+ */
+ ~UserSpecifiedAZeroRefreshInterval() override;
+}; // class UserSpecifiedAZeroRefreshInterval
+
+} // namespace catalogue
+} // namespace cta
diff --git a/catalogue/UserSpecifiedAZeroTargetedFreeSpace.cpp b/catalogue/UserSpecifiedAZeroTargetedFreeSpace.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..550de7ef660e1e797642c98d66904097f281e2e2
--- /dev/null
+++ b/catalogue/UserSpecifiedAZeroTargetedFreeSpace.cpp
@@ -0,0 +1,39 @@
+/*
+ * The CERN Tape Archive (CTA) project
+ * Copyright (C) 2015 CERN
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "catalogue/UserSpecifiedAZeroTargetedFreeSpace.hpp"
+
+namespace cta {
+namespace catalogue {
+
+
+//------------------------------------------------------------------------------
+// constructor
+//------------------------------------------------------------------------------
+UserSpecifiedAZeroTargetedFreeSpace::UserSpecifiedAZeroTargetedFreeSpace(const std::string &context, const bool embedBacktrace):
+ cta::exception::UserError(context, embedBacktrace) {
+}
+
+//------------------------------------------------------------------------------
+// destructor
+//------------------------------------------------------------------------------
+UserSpecifiedAZeroTargetedFreeSpace::~UserSpecifiedAZeroTargetedFreeSpace() {
+}
+
+} // namespace catalogue
+} // namespace cta
diff --git a/catalogue/UserSpecifiedAZeroTargetedFreeSpace.hpp b/catalogue/UserSpecifiedAZeroTargetedFreeSpace.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..f5d8cc3444973852912d51ced9a67a0ca6cc0db1
--- /dev/null
+++ b/catalogue/UserSpecifiedAZeroTargetedFreeSpace.hpp
@@ -0,0 +1,48 @@
+/*
+ * The CERN Tape Archive (CTA) project
+ * Copyright (C) 2015 CERN
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include "common/exception/UserError.hpp"
+
+namespace cta {
+namespace catalogue {
+
+/**
+ * User specified a zero value for a targeted free space when this is not permitted.
+ */
+class UserSpecifiedAZeroTargetedFreeSpace: public exception::UserError {
+public:
+ /**
+ * Constructor.
+ *
+ * @param context optional context string added to the message
+ * at initialisation time.
+ * @param embedBacktrace whether to embed a backtrace of where the
+ * exception was throw in the message
+ */
+ UserSpecifiedAZeroTargetedFreeSpace(const std::string &context = "", const bool embedBacktrace = true);
+
+ /**
+ * Destructor.
+ */
+ ~UserSpecifiedAZeroTargetedFreeSpace() override;
+}; // class UserSpecifiedAZeroTargetedFreeSpace
+
+} // namespace catalogue
+} // namespace cta
diff --git a/catalogue/UserSpecifiedAnEmptyStringDiskSystemName.cpp b/catalogue/UserSpecifiedAnEmptyStringDiskSystemName.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..c814f346755b1f9bab77737249055e376b4dc4a2
--- /dev/null
+++ b/catalogue/UserSpecifiedAnEmptyStringDiskSystemName.cpp
@@ -0,0 +1,39 @@
+/*
+ * The CERN Tape Archive (CTA) project
+ * Copyright (C) 2015 CERN
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "catalogue/UserSpecifiedAnEmptyStringDiskSystemName.hpp"
+
+namespace cta {
+namespace catalogue {
+
+
+//------------------------------------------------------------------------------
+// constructor
+//------------------------------------------------------------------------------
+UserSpecifiedAnEmptyStringDiskSystemName::UserSpecifiedAnEmptyStringDiskSystemName(const std::string &context, const bool embedBacktrace):
+ cta::exception::UserError(context, embedBacktrace) {
+}
+
+//------------------------------------------------------------------------------
+// destructor
+//------------------------------------------------------------------------------
+UserSpecifiedAnEmptyStringDiskSystemName::~UserSpecifiedAnEmptyStringDiskSystemName() {
+}
+
+} // namespace catalogue
+} // namespace cta
diff --git a/catalogue/UserSpecifiedAnEmptyStringDiskSystemName.hpp b/catalogue/UserSpecifiedAnEmptyStringDiskSystemName.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..efbc2d811c8983f01cc8e853351243e225c93c7e
--- /dev/null
+++ b/catalogue/UserSpecifiedAnEmptyStringDiskSystemName.hpp
@@ -0,0 +1,48 @@
+/*
+ * The CERN Tape Archive (CTA) project
+ * Copyright (C) 2015 CERN
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include "common/exception/UserError.hpp"
+
+namespace cta {
+namespace catalogue {
+
+/**
+ * User specified an empty string for a disk system name when this is not permitted.
+ */
+class UserSpecifiedAnEmptyStringDiskSystemName: public exception::UserError {
+public:
+ /**
+ * Constructor.
+ *
+ * @param context optional context string added to the message
+ * at initialisation time.
+ * @param embedBacktrace whether to embed a backtrace of where the
+ * exception was throw in the message
+ */
+ UserSpecifiedAnEmptyStringDiskSystemName(const std::string &context = "", const bool embedBacktrace = true);
+
+ /**
+ * Destructor.
+ */
+ ~UserSpecifiedAnEmptyStringDiskSystemName() override;
+}; // class UserSpecifiedAnEmptyStringDiskSystemName
+
+} // namespace catalogue
+} // namespace cta
diff --git a/catalogue/UserSpecifiedAnEmptyStringFileRegexp.cpp b/catalogue/UserSpecifiedAnEmptyStringFileRegexp.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..331d34c8c7d01971e8ce518cbb93e2600a414079
--- /dev/null
+++ b/catalogue/UserSpecifiedAnEmptyStringFileRegexp.cpp
@@ -0,0 +1,39 @@
+/*
+ * The CERN Tape Archive (CTA) project
+ * Copyright (C) 2015 CERN
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "catalogue/UserSpecifiedAnEmptyStringFileRegexp.hpp"
+
+namespace cta {
+namespace catalogue {
+
+
+//------------------------------------------------------------------------------
+// constructor
+//------------------------------------------------------------------------------
+UserSpecifiedAnEmptyStringFileRegexp::UserSpecifiedAnEmptyStringFileRegexp(const std::string &context, const bool embedBacktrace):
+ cta::exception::UserError(context, embedBacktrace) {
+}
+
+//------------------------------------------------------------------------------
+// destructor
+//------------------------------------------------------------------------------
+UserSpecifiedAnEmptyStringFileRegexp::~UserSpecifiedAnEmptyStringFileRegexp() {
+}
+
+} // namespace catalogue
+} // namespace cta
diff --git a/catalogue/UserSpecifiedAnEmptyStringFileRegexp.hpp b/catalogue/UserSpecifiedAnEmptyStringFileRegexp.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..c1899e48cc2c5475644d112621adb25eef6e986d
--- /dev/null
+++ b/catalogue/UserSpecifiedAnEmptyStringFileRegexp.hpp
@@ -0,0 +1,48 @@
+/*
+ * The CERN Tape Archive (CTA) project
+ * Copyright (C) 2015 CERN
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include "common/exception/UserError.hpp"
+
+namespace cta {
+namespace catalogue {
+
+/**
+ * User specified an empty string for a file regexp when this is not permitted.
+ */
+class UserSpecifiedAnEmptyStringFileRegexp: public exception::UserError {
+public:
+ /**
+ * Constructor.
+ *
+ * @param context optional context string added to the message
+ * at initialisation time.
+ * @param embedBacktrace whether to embed a backtrace of where the
+ * exception was throw in the message
+ */
+ UserSpecifiedAnEmptyStringFileRegexp(const std::string &context = "", const bool embedBacktrace = true);
+
+ /**
+ * Destructor.
+ */
+ ~UserSpecifiedAnEmptyStringFileRegexp() override;
+}; // class UserSpecifiedAnEmptyStringFileRegexp
+
+} // namespace catalogue
+} // namespace cta
diff --git a/catalogue/UserSpecifiedAnEmptyStringFreeSpaceQueryURL.cpp b/catalogue/UserSpecifiedAnEmptyStringFreeSpaceQueryURL.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..9d1eb07ae92e2265a5acec1adb7287437101afa0
--- /dev/null
+++ b/catalogue/UserSpecifiedAnEmptyStringFreeSpaceQueryURL.cpp
@@ -0,0 +1,39 @@
+/*
+ * The CERN Tape Archive (CTA) project
+ * Copyright (C) 2015 CERN
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "catalogue/UserSpecifiedAnEmptyStringFreeSpaceQueryURL.hpp"
+
+namespace cta {
+namespace catalogue {
+
+
+//------------------------------------------------------------------------------
+// constructor
+//------------------------------------------------------------------------------
+UserSpecifiedAnEmptyStringFreeSpaceQueryURL::UserSpecifiedAnEmptyStringFreeSpaceQueryURL(const std::string &context, const bool embedBacktrace):
+ cta::exception::UserError(context, embedBacktrace) {
+}
+
+//------------------------------------------------------------------------------
+// destructor
+//------------------------------------------------------------------------------
+UserSpecifiedAnEmptyStringFreeSpaceQueryURL::~UserSpecifiedAnEmptyStringFreeSpaceQueryURL() {
+}
+
+} // namespace catalogue
+} // namespace cta
diff --git a/catalogue/UserSpecifiedAnEmptyStringFreeSpaceQueryURL.hpp b/catalogue/UserSpecifiedAnEmptyStringFreeSpaceQueryURL.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..3fb8d674ac6c0266b103e36548789f43ba33c15c
--- /dev/null
+++ b/catalogue/UserSpecifiedAnEmptyStringFreeSpaceQueryURL.hpp
@@ -0,0 +1,48 @@
+/*
+ * The CERN Tape Archive (CTA) project
+ * Copyright (C) 2015 CERN
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include "common/exception/UserError.hpp"
+
+namespace cta {
+namespace catalogue {
+
+/**
+ * User specified an empty string for a free space query URL when this is not permitted.
+ */
+class UserSpecifiedAnEmptyStringFreeSpaceQueryURL: public exception::UserError {
+public:
+ /**
+ * Constructor.
+ *
+ * @param context optional context string added to the message
+ * at initialisation time.
+ * @param embedBacktrace whether to embed a backtrace of where the
+ * exception was throw in the message
+ */
+ UserSpecifiedAnEmptyStringFreeSpaceQueryURL(const std::string &context = "", const bool embedBacktrace = true);
+
+ /**
+ * Destructor.
+ */
+ ~UserSpecifiedAnEmptyStringFreeSpaceQueryURL() override;
+}; // class UserSpecifiedAnEmptyStringFreeSpaceQueryURL
+
+} // namespace catalogue
+} // namespace cta
diff --git a/catalogue/common_catalogue_schema.sql b/catalogue/common_catalogue_schema.sql
index d6614edcf0b6648f865912840b1aca5a1ea4262f..b848cea07c2f03e3f5cf244987dda0e5b8b615f0 100644
--- a/catalogue/common_catalogue_schema.sql
+++ b/catalogue/common_catalogue_schema.sql
@@ -13,6 +13,22 @@ CREATE TABLE ADMIN_USER(
LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT ADMIN_USER_LUT_NN NOT NULL,
CONSTRAINT ADMIN_USER_PK PRIMARY KEY(ADMIN_USER_NAME)
);
+CREATE TABLE DISK_SYSTEM(
+ DISK_SYSTEM_NAME VARCHAR(100) CONSTRAINT DISK_SYSTEM_DSNM_NN NOT NULL,
+ FILE_REGEXP VARCHAR(100) CONSTRAINT DISK_SYSTEM_FR_NN NOT NULL,
+ FREE_SPACE_QUERY_URL VARCHAR(1000) CONSTRAINT DISK_SYSTEM_FSQU_NN NOT NULL,
+ REFRESH_INTERVAL NUMERIC(20, 0) CONSTRAINT DISK_SYSTEM_RI_NN NOT NULL,
+ TARGETED_FREE_SPACE NUMERIC(20, 0) CONSTRAINT DISK_SYSTEM_TFS_NN NOT NULL,
+ SLEEP_TIME NUMERIC(20, 0) CONSTRAINT DISK_SYSTEM_ST_NN NOT NULL,
+ USER_COMMENT VARCHAR(1000) CONSTRAINT DISK_SYSTEM_UC_NN NOT NULL,
+ CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT DISK_SYSTEM_CLUN_NN NOT NULL,
+ CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT DISK_SYSTEM_CLHN_NN NOT NULL,
+ CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT DISK_SYSTEM_CLT_NN NOT NULL,
+ LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT DISK_SYSTEM_LUUN_NN NOT NULL,
+ LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT DISK_SYSTEM_LUHN_NN NOT NULL,
+ LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT DISK_SYSTEM_LUT_NN NOT NULL,
+ CONSTRAINT NAME_PK PRIMARY KEY(DISK_SYSTEM_NAME)
+);
CREATE TABLE STORAGE_CLASS(
STORAGE_CLASS_ID NUMERIC(20, 0) CONSTRAINT STORAGE_CLASS_SCI_NN NOT NULL,
DISK_INSTANCE_NAME VARCHAR(100) CONSTRAINT STORAGE_CLASS_DIN_NN NOT NULL,
diff --git a/cmake/UseRPMTools.cmake b/cmake/UseRPMTools.cmake
index b22729a03ec95c4be79792155322897a62bff867..2ab6553d4fe00a0df75052d1f4bf826b6a26007b 100644
--- a/cmake/UseRPMTools.cmake
+++ b/cmake/UseRPMTools.cmake
@@ -27,7 +27,7 @@ IF (UNIX)
DOC "The RPM builder tool")
IF (RPMTools_RPMBUILD_EXECUTABLE)
- MESSAGE(STATUS "Looking for RPMTools... - found rpmuild is ${RPMTools_RPMBUILD_EXECUTABLE}")
+ MESSAGE(STATUS "Looking for RPMTools... - found rpmbuild in ${RPMTools_RPMBUILD_EXECUTABLE}")
SET(RPMTools_RPMBUILD_FOUND "YES")
GET_FILENAME_COMPONENT(RPMTools_BINARY_DIRS ${RPMTools_RPMBUILD_EXECUTABLE} PATH)
ELSE (RPMTools_RPMBUILD_EXECUTABLE)
diff --git a/cmdline/CtaAdminCmd.cpp b/cmdline/CtaAdminCmd.cpp
index a65c5bfa02c10c206ee9aaad4191c936684bdedc..5950259823c26700a2e8f47cadb48644416da0ba 100644
--- a/cmdline/CtaAdminCmd.cpp
+++ b/cmdline/CtaAdminCmd.cpp
@@ -91,6 +91,7 @@ void IStreamBuffer<cta::xrd::Data>::DataCallback(cta::xrd::Data record) const
case Data::kSclsItem: std::cout << Log::DumpProtobuf(&record.scls_item()); break;
case Data::kTalsItem: std::cout << Log::DumpProtobuf(&record.tals_item()); break;
case Data::kTplsItem: std::cout << Log::DumpProtobuf(&record.tpls_item()); break;
+ case Data::kDslsItem: std::cout << Log::DumpProtobuf(&record.dsls_item()); break;
default:
throw std::runtime_error("Received invalid stream data from CTA Frontend.");
}
@@ -117,6 +118,7 @@ void IStreamBuffer<cta::xrd::Data>::DataCallback(cta::xrd::Data record) const
case Data::kSclsItem: formattedText.print(record.scls_item()); break;
case Data::kTalsItem: formattedText.print(record.tals_item()); break;
case Data::kTplsItem: formattedText.print(record.tpls_item()); break;
+ case Data::kDslsItem: formattedText.print(record.dsls_item()); break;
default:
throw std::runtime_error("Received invalid stream data from CTA Frontend.");
}
@@ -270,6 +272,7 @@ void CtaAdminCmd::send() const
case HeaderType::STORAGECLASS_LS: formattedText.printStorageClassLsHeader(); break;
case HeaderType::TAPE_LS: formattedText.printTapeLsHeader(); break;
case HeaderType::TAPEPOOL_LS: formattedText.printTapePoolLsHeader(); break;
+ case HeaderType::DISKSYSTEM_LS: formattedText.printDiskSystemLsHeader(); break;
case HeaderType::NONE:
default: break;
}
diff --git a/cmdline/CtaAdminCmdParse.hpp b/cmdline/CtaAdminCmdParse.hpp
index 23e8316ec7410b14b6223f33f96a9cdc90845b4e..fa080f80849714aeff28ca1d95b1d1ec7a57ce17 100644
--- a/cmdline/CtaAdminCmdParse.hpp
+++ b/cmdline/CtaAdminCmdParse.hpp
@@ -211,6 +211,8 @@ const cmdLookup_t cmdLookup = {
{ "ta", AdminCmd::CMD_TAPE },
{ "tapepool", AdminCmd::CMD_TAPEPOOL },
{ "tp", AdminCmd::CMD_TAPEPOOL },
+ { "disksystem", AdminCmd::CMD_DISKSYSTEM },
+ { "ds", AdminCmd::CMD_DISKSYSTEM }
};
@@ -279,6 +281,9 @@ const std::map<std::string, OptionUInt64::Key> uint64Options = {
{ "--partialtapesnumber", OptionUInt64::PARTIAL_TAPES_NUMBER },
{ "--retrievepriority", OptionUInt64::RETRIEVE_PRIORITY },
{ "--size", OptionUInt64::FILE_SIZE },
+ { "--refreshinterval", OptionUInt64::REFRESH_INTERVAL },
+ { "--targetedfreespace", OptionUInt64::TARGETED_FREE_SPACE },
+ { "--sleeptime", OptionUInt64::SLEEP_TIME },
{ "--uid", OptionUInt64::OWNER_UID }
};
@@ -308,7 +313,10 @@ const std::map<std::string, OptionString::Key> strOptions = {
{ "--username", OptionString::USERNAME },
{ "--vendor", OptionString::VENDOR },
{ "--vid", OptionString::VID },
- { "--vo", OptionString::VO }
+ { "--vo", OptionString::VO },
+ { "--disksystem", OptionString::DISK_SYSTEM },
+ { "--fileregexp", OptionString::FILE_REGEXP },
+ { "--freespacequeryurl", OptionString::FREE_SPACE_QUERY_URL }
};
@@ -360,6 +368,7 @@ const std::map<AdminCmd::Cmd, CmdHelp> cmdHelp = {
{ AdminCmd::CMD_STORAGECLASS, { "storageclass", "sc", { "add", "ch", "rm", "ls" } }},
{ AdminCmd::CMD_TAPE, { "tape", "ta", { "add", "ch", "rm", "reclaim", "ls", "label" } }},
{ AdminCmd::CMD_TAPEPOOL, { "tapepool", "tp", { "add", "ch", "rm", "ls" } }},
+ { AdminCmd::CMD_DISKSYSTEM, { "disksystem", "ds", { "add", "ch", "rm", "ls" } }},
};
@@ -436,7 +445,12 @@ const Option opt_full { Option::OPT_BOOL, "--full",
const Option opt_readonly { Option::OPT_BOOL, "--readonly", "-r", " <\"true\" or \"false\">" };
const Option opt_disabled_tape { Option::OPT_FLAG, "--disabledtape", "-d", ""};
-
+const Option opt_disksystem { Option::OPT_STR, "--disksystem", "-n", " <disk_system_name>" };
+const Option opt_file_regexp { Option::OPT_STR, "--fileregexp", "-r", " <file_regexp>" };
+const Option opt_free_space_query_url { Option::OPT_STR, "--freespacequeryurl", "-u", " <free_space_query_url>" };
+const Option opt_refresh_interval { Option::OPT_UINT, "--refreshinterval", "-i", " <refresh_intreval>" };
+const Option opt_targeted_free_space { Option::OPT_UINT, "--targetedfreespace", "-f", " <targeted_free_space>" };
+const Option opt_sleep_time { Option::OPT_UINT, "--sleeptime", "-s", " <sleep time in s>" };
/*!
* Map valid options to commands
@@ -547,6 +561,15 @@ const std::map<cmd_key_t, cmd_val_t> cmdOptions = {
opt_supply.optional(), opt_comment.optional() }},
{{ AdminCmd::CMD_TAPEPOOL, AdminCmd::SUBCMD_RM }, { opt_tapepool_alias }},
{{ AdminCmd::CMD_TAPEPOOL, AdminCmd::SUBCMD_LS }, { }},
+ /*----------------------------------------------------------------------------------------------------*/
+ {{ AdminCmd::CMD_DISKSYSTEM, AdminCmd::SUBCMD_ADD },
+ { opt_disksystem, opt_file_regexp, opt_free_space_query_url, opt_refresh_interval, opt_targeted_free_space, opt_sleep_time,
+ opt_comment }},
+ {{ AdminCmd::CMD_DISKSYSTEM, AdminCmd::SUBCMD_CH },
+ { opt_disksystem, opt_file_regexp.optional(), opt_free_space_query_url.optional(), opt_refresh_interval.optional(),
+ opt_targeted_free_space.optional(), opt_sleep_time.optional(), opt_comment.optional() }},
+ {{ AdminCmd::CMD_DISKSYSTEM, AdminCmd::SUBCMD_RM }, { opt_disksystem }},
+ {{ AdminCmd::CMD_DISKSYSTEM, AdminCmd::SUBCMD_LS }, { }},
};
diff --git a/cmdline/CtaAdminTextFormatter.cpp b/cmdline/CtaAdminTextFormatter.cpp
index 7a0b23a74737969c2fbcb178f00340afe52c5198..810c0c8a3de5362633dbf446d3cccc2676556a78 100644
--- a/cmdline/CtaAdminTextFormatter.cpp
+++ b/cmdline/CtaAdminTextFormatter.cpp
@@ -894,4 +894,42 @@ void TextFormatter::print(const TapePoolLsItem &tpls_item)
);
}
+void TextFormatter::printDiskSystemLsHeader() {
+ push_back("HEADER");
+ push_back(
+ "name",
+ "regexp",
+ "url",
+ "refresh",
+ "space",
+ "sleep",
+ "c.user",
+ "c.host",
+ "c.time",
+ "m.user",
+ "m.host",
+ "m.time",
+ "comment"
+ );
+}
+
+void TextFormatter::print(const DiskSystemLsItem &dsls_item)
+{
+ push_back(
+ dsls_item.name(),
+ dsls_item.file_regexp(),
+ dsls_item.free_space_query_url(),
+ dsls_item.refresh_interval(),
+ dsls_item.targeted_free_space(),
+ dsls_item.sleep_time(),
+ dsls_item.creation_log().username(),
+ dsls_item.creation_log().host(),
+ timeToStr(dsls_item.creation_log().time()),
+ dsls_item.last_modification_log().username(),
+ dsls_item.last_modification_log().host(),
+ timeToStr(dsls_item.last_modification_log().time()),
+ dsls_item.comment()
+ );
+}
+
}}
diff --git a/cmdline/CtaAdminTextFormatter.hpp b/cmdline/CtaAdminTextFormatter.hpp
index 533e534f5d414e4a322175f45890fbf57a20d1a0..f8acc3d8ad78577c5bd6029781da08d68d766397 100644
--- a/cmdline/CtaAdminTextFormatter.hpp
+++ b/cmdline/CtaAdminTextFormatter.hpp
@@ -64,6 +64,7 @@ public:
void printStorageClassLsHeader();
void printTapeLsHeader();
void printTapePoolLsHeader();
+ void printDiskSystemLsHeader();
// Output records
void print(const AdminLsItem &adls_item);
@@ -86,6 +87,8 @@ public:
void print(const StorageClassLsItem &scls_item);
void print(const TapeLsItem &tals_item);
void print(const TapePoolLsItem &tpls_item);
+ void print(const DiskSystemLsItem &dsls_item);
+
private:
//! Add a line to the buffer
diff --git a/common/dataStructures/ArchiveFile.hpp b/common/dataStructures/ArchiveFile.hpp
index 73cb3c5eadac1e1d3b8e6cb605d5a93595e2a9af..76e56c274a41ff38228ab73bda71ce1f91e4eb1c 100644
--- a/common/dataStructures/ArchiveFile.hpp
+++ b/common/dataStructures/ArchiveFile.hpp
@@ -57,7 +57,7 @@ struct ArchiveFile {
std::string storageClass;
DiskFileInfo diskFileInfo;
/**
- * This map represents the non-necessarily-exhaustive set of tape copies
+ * This list represents the non-necessarily-exhaustive set of tape copies
* to be listed by the operator. For example, if the listing requested is
* for a single tape, the map will contain only one element.
*/
diff --git a/common/dataStructures/QueueAndMountSummary.hpp b/common/dataStructures/QueueAndMountSummary.hpp
index 685e4ba5cd8673bf323d53767befa45407212ca8..b6ef864047b88ba46c1eddae0fb52f4aa7eee6ec 100644
--- a/common/dataStructures/QueueAndMountSummary.hpp
+++ b/common/dataStructures/QueueAndMountSummary.hpp
@@ -52,6 +52,12 @@ struct QueueAndMountSummary {
uint64_t emptyTapes=0;
uint64_t disabledTapes=0;
uint64_t writableTapes=0;
+ struct SleepForSpaceInfo {
+ time_t startTime;
+ std::string diskSystemName;
+ time_t sleepTime;
+ };
+ optional<SleepForSpaceInfo> sleepForSpaceInfo;
static QueueAndMountSummary &getOrCreateEntry(std::list<QueueAndMountSummary> &summaryList,
MountType mountType, const std::string &tapePool, const std::string &vid,
diff --git a/common/utils/Regex.cpp b/common/utils/Regex.cpp
index b3fa149fc33721ad65b14dd1f36bfbafdfdf14d1..60a91ce24e797c9e28c5b71f79978db0d3a4a7b3 100644
--- a/common/utils/Regex.cpp
+++ b/common/utils/Regex.cpp
@@ -27,10 +27,10 @@
namespace cta { namespace utils {
-Regex::Regex(const char * re_str) : m_set(false) {
- if (int rc = ::regcomp(&m_re, re_str, REG_EXTENDED)) {
+Regex::Regex(const std::string & re_str) : m_reStr(re_str), m_set(false) {
+ if (int rc = ::regcomp(&m_re, m_reStr.c_str(), REG_EXTENDED)) {
std::string error("Could not compile regular expression: \"");
- error += re_str;
+ error += m_reStr;
error += "\"";
char re_err[1024];
if (::regerror(rc, &m_re, re_err, sizeof (re_err))) {
@@ -42,12 +42,16 @@ Regex::Regex(const char * re_str) : m_set(false) {
m_set = true;
}
+Regex::Regex(const Regex& o) {
+ Regex(o.m_reStr);
+}
+
Regex::~Regex() {
if (m_set)
::regfree(&m_re);
}
-std::vector<std::string> Regex::exec(const std::string &s) {
+std::vector<std::string> Regex::exec(const std::string &s) const {
regmatch_t matches[100];
if (REG_NOMATCH != ::regexec(&m_re, s.c_str(), 100, matches, 0)) {
std::vector<std::string> ret;
diff --git a/common/utils/Regex.hpp b/common/utils/Regex.hpp
index 834a54d8f8ef10b8501b2cbceb8a67cad4be64e2..5b5bf259d72a88d159f9c253345a7a8fdff418aa 100644
--- a/common/utils/Regex.hpp
+++ b/common/utils/Regex.hpp
@@ -29,13 +29,14 @@ namespace cta { namespace utils {
class Regex {
public:
- Regex(const char * re_str);
+ Regex(const std::string & re_str);
+ Regex(const Regex & o);
virtual ~Regex();
/*!
* Return a list of matching substrings
*/
- std::vector<std::string> exec(const std::string &s);
+ std::vector<std::string> exec(const std::string &s) const;
/*!
* Return true if there is at least one matching substring
@@ -45,6 +46,8 @@ class Regex {
}
private:
+ // We keep around the string from which the RE was compiled to allow copying.
+ std::string m_reStr;
regex_t m_re;
bool m_set;
};
diff --git a/continuousintegration/buildtree_runner/recreate_buildtree_running_environment.sh b/continuousintegration/buildtree_runner/recreate_buildtree_running_environment.sh
index ed722256e5a5305d473d6b0c5b2ceafffb9122e5..4cdff33c6df793574097c1678d0dd7db851bb647 100755
--- a/continuousintegration/buildtree_runner/recreate_buildtree_running_environment.sh
+++ b/continuousintegration/buildtree_runner/recreate_buildtree_running_environment.sh
@@ -73,3 +73,9 @@ mkdir -pv /shared/stg
sudo kubectl delete pv stg || true
sudo kubectl create -f ./stg_PV.yaml
sudo kubectl get persistentvolumes -l type=stg
+
+echo Generating the share host path
+rm -rf /shared/cta
+mkdir -pv /shared/cta/catdb
+touch /shared/cta/catdb/catdb
+chmod 1777 /shared/cta
diff --git a/continuousintegration/buildtree_runner/vmBootstrap/bootstrapCTA.sh b/continuousintegration/buildtree_runner/vmBootstrap/bootstrapCTA.sh
index 7ea4bf7496b0faf839c16c8a5ffaf8f0656a3742..219ed85fe634c48079ff3d54a9d11eaa7ffad4eb 100755
--- a/continuousintegration/buildtree_runner/vmBootstrap/bootstrapCTA.sh
+++ b/continuousintegration/buildtree_runner/vmBootstrap/bootstrapCTA.sh
@@ -14,13 +14,21 @@ chmod +x ~/CTA/.git/hooks/post-checkout
cp ~/CTA/.git/hooks/post-checkout ~/CTA/.git/hooks/post-merge
echo Creating source rpm
+rm -rf ~/CTA-build-srpm
mkdir -p ~/CTA-build-srpm
(cd ~/CTA-build-srpm && cmake -DPackageOnly:Bool=true ../CTA; make cta_srpm)
echo Installing repos
+<<<<<<< HEAD
+for r in `ls -1 ../../docker/ctafrontend/cc7/etc/yum.repos.d/*.repo`; do
+ yum-config-manager --add-repo=$r
+done
+sudo yum-config-manager --add-repo=ceph-internal.repo
+=======
for r in `ls ../../docker/ctafrontend/cc7/etc/yum.repos.d/*.repo` ceph-internal.repo; do
sudo yum-config-manager --add-repo=$r
done
+>>>>>>> origin/master
sudo yum install -y yum-plugin-priorities
echo Adding versionlock for xrootd:
diff --git a/continuousintegration/orchestration/tests/repack_systemtest.sh b/continuousintegration/orchestration/tests/repack_systemtest.sh
index b206f30c7617cbafa1c47ecc290df22e90896dff..dd6d8c01cdd3464cdc915c18c160ad15493da5e3 100755
--- a/continuousintegration/orchestration/tests/repack_systemtest.sh
+++ b/continuousintegration/orchestration/tests/repack_systemtest.sh
@@ -112,10 +112,38 @@ admin_cta repack rm --vid ${VID_TO_REPACK}
echo "Marking the tape ${VID_TO_REPACK} as full before Repacking it"
admin_cta tape ch --vid ${VID_TO_REPACK} --full true
+echo "Backpressure test: setting too high free space requirements"
+# This should be idempotent as we will be called several times
+if [[ $( admin_cta --json ds ls | jq '.[] | select(.name=="repackBuffer") | .name') != '"repackBuffer"' ]]; then
+ admin_cta ds add -n repackBuffer -r "root://${EOSINSTANCE}/${REPACK_BUFFER_BASEDIR}" -u "eos://${EOSINSTANCE}" -i 5 -f 111222333444555 -s 60 -m toto
+else
+ echo "Disk system repackBuffer alread defined. Ensuring too high free space requirements."
+ admin_cta ds ch -n repackBuffer -f 111222333444555
+fi
+admin_cta ds ls
+
echo "Launching repack request for VID ${VID_TO_REPACK}, bufferURL = ${FULL_REPACK_BUFFER_URL}"
admin_cta repack add --vid ${VID_TO_REPACK} ${REPACK_OPTION} --bufferurl ${FULL_REPACK_BUFFER_URL} ${DISABLED_TAPE_FLAG}
+echo "Backpressure test: waiting to see a report of sleeping retrieve queue."
+SECONDS_PASSED=0
+while test 0 = `admin_cta --json sq | jq -r ".[] | select(.vid == \"${VID_TO_REPACK}\" and .sleepingForSpace == true) | .vid" | wc -l`; do
+ echo "Waiting for retrieve queue for tape ${VID_TO_REPACK} to be sleepting: Seconds passed = $SECONDS_PASSED"
+ sleep 1
+ let SECONDS_PASSED=SECONDS_PASSED+1
+
+ if test ${SECONDS_PASSED} == ${WAIT_FOR_REPACK_TIMEOUT}; then
+ echo "Timed out after ${WAIT_FOR_REPACK_TIMEOUT} seconds waiting for tape ${VID_TO_REPACK} to be repacked"
+ exit 1
+ fi
+done
+
+echo "Turning free space requirement to one byte (zero is not allowed)."
+admin_cta ds ch -n repackBuffer -f 1
+admin_cta ds ls
+
+echo "Now waiting for repack to proceed."
SECONDS_PASSED=0
while test 0 = `admin_cta --json repack ls --vid ${VID_TO_REPACK} | jq -r '.[0] | select(.status == "Complete" or .status == "Failed")' | wc -l`; do
echo "Waiting for repack request on tape ${VID_TO_REPACK} to be complete: Seconds passed = $SECONDS_PASSED"
@@ -140,4 +168,4 @@ echo "Repack request on VID ${VID_TO_REPACK} succeeded."
exec /root/repack_generate_report.sh -v ${VID_TO_REPACK} -r ${REPORT_DIRECTORY} ${ADD_COPIES_ONLY} &
wait $!
-exit 0
\ No newline at end of file
+exit 0
diff --git a/cta.spec.in b/cta.spec.in
index de99a3eba8c738bed0e638bc03c704ade6c00732..922c8fbf1a08f17fb445b110200513ce555d553d 100644
--- a/cta.spec.in
+++ b/cta.spec.in
@@ -37,10 +37,10 @@ BuildRequires: cmake >= 2.6 redhat-rpm-config
# The CTA client is the only component of CTA that can be compiled on both SLC6
# and C77, therefore only the packages it depends on are required for SLC6
%define radosVersion 2:12.2.2
-BuildRequires: xrootd-client-devel >= 1:4.8.2
-BuildRequires: xrootd-devel >= 1:4.8.2
-BuildRequires: xrootd-server-devel >= 1:4.8.2
-BuildRequires: xrootd-private-devel >= 1:4.8.2
+BuildRequires: xrootd-client-devel >= 1:4.10.0
+BuildRequires: xrootd-devel >= 1:4.10.0
+BuildRequires: xrootd-server-devel >= 1:4.10.0
+BuildRequires: xrootd-private-devel >= 1:4.10.0
BuildRequires: librados-devel = %{radosVersion}, libradosstriper-devel = %{radosVersion},
BuildRequires: protobuf3-compiler >= 3.3.1 protobuf3-devel >= 3.3.1
BuildRequires: gmock-devel >= 1.5.0 gtest-devel >= 1.5.0
@@ -110,6 +110,7 @@ Group: Application/CTA
Requires: logrotate
Requires: cta-common = %{version}-%{release}
Requires: cta-lib = %{version}-%{release}
+Requires: eos-client
%description -n cta-taped
CERN Tape Archive:
The tape server daemon
diff --git a/disk/CMakeLists.txt b/disk/CMakeLists.txt
index 7bb142594d7018e4ce92305dc699c3a8076e46a9..b29af87d1f611606f05b00b8a293c568274fb5f5 100644
--- a/disk/CMakeLists.txt
+++ b/disk/CMakeLists.txt
@@ -27,6 +27,7 @@ add_library(ctadisk SHARED
EOSReporter.cpp
DiskFile.cpp
RadosStriperPool.cpp
+ DiskSystem.cpp
)
target_link_libraries (ctadisk XrdCl cryptopp radosstriper)
@@ -42,4 +43,4 @@ set_property(TARGET ctadiskunittests PROPERTY SOVERSION "${CTA_SOVERSION}")
set_property(TARGET ctadiskunittests PROPERTY VERSION "${CTA_LIBVERSION}")
install(TARGETS ctadisk DESTINATION usr/${CMAKE_INSTALL_LIBDIR})
-install(TARGETS ctadiskunittests DESTINATION usr/${CMAKE_INSTALL_LIBDIR})
\ No newline at end of file
+install(TARGETS ctadiskunittests DESTINATION usr/${CMAKE_INSTALL_LIBDIR})
diff --git a/disk/DiskSystem.cpp b/disk/DiskSystem.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..6453b4d9e9889b74ce7182908562f6988a28eacc
--- /dev/null
+++ b/disk/DiskSystem.cpp
@@ -0,0 +1,148 @@
+/*
+ * The CERN Tape Archive (CTA) project
+ * Copyright (C) 2015 CERN
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "DiskSystem.hpp"
+
+#include <algorithm>
+#include "common/exception/Exception.hpp"
+#include "common/threading/SubProcess.hpp"
+#include "common/exception/Errnum.hpp"
+#include "common/utils/utils.hpp"
+
+namespace cta {
+namespace disk {
+
+//------------------------------------------------------------------------------
+// DiskSystemList::at()
+//------------------------------------------------------------------------------
+const DiskSystem& DiskSystemList::at(const std::string& name) const {
+ auto dsi = std::find_if(begin(), end(), [&](const DiskSystem& ds){ return ds.name == name;});
+ if (dsi != end()) return *dsi;
+ throw std::out_of_range("In DiskSystemList::at(): name not found.");
+}
+
+//------------------------------------------------------------------------------
+// DiskSystemList::getFSNAme()
+//------------------------------------------------------------------------------
+std::string DiskSystemList::getDSNAme(const std::string& fileURL) const {
+ // First if the regexes have not been created yet, do so.
+ if (m_pointersAndRegexes.empty() && size()) {
+ for (const auto &ds: *this) {
+ m_pointersAndRegexes.emplace_back(ds, ds.fileRegexp);
+ }
+ }
+ // Try and find the fileURL
+ auto pri = std::find_if(m_pointersAndRegexes.begin(), m_pointersAndRegexes.end(),
+ [&](const PointerAndRegex & pr){ return !pr.regex.exec(fileURL).empty(); });
+ if (pri != m_pointersAndRegexes.end()) {
+ // We found a match. Let's move the pointer and regex to the front so next file will be faster (most likely).
+ if (pri != m_pointersAndRegexes.begin())
+ m_pointersAndRegexes.splice(m_pointersAndRegexes.begin(), m_pointersAndRegexes, pri);
+ return pri->ds.name;
+ }
+ throw std::out_of_range("In DiskSystemList::getFSNAme(): not match for fileURL");
+}
+
+//------------------------------------------------------------------------------
+// DiskSystemFreeSpaceList::fetchFileSystemFreeSpace()
+//------------------------------------------------------------------------------
+void DiskSystemFreeSpaceList::fetchDiskSystemFreeSpace(const std::set<std::string>& diskSystems, log::LogContext & lc) {
+ // The real deal: go fetch the file system's free space.
+ cta::utils::Regex eosDiskSystem("^eos://(.*)$");
+ // For testing purposes
+ cta::utils::Regex constantFreeSpaceDiskSystem("^constantFreeSpace:(.*)");
+ for (auto const & ds: diskSystems) {
+ uint64_t freeSpace;
+ std::vector<std::string> regexResult;
+ regexResult = eosDiskSystem.exec(m_systemList.at(ds).freeSpaceQueryURL);
+ if (regexResult.size()) {
+ freeSpace = fetchEosFreeSpace(regexResult.at(1), lc);
+ goto found;
+ }
+ regexResult = constantFreeSpaceDiskSystem.exec(m_systemList.at(ds).freeSpaceQueryURL);
+ if (regexResult.size()) {
+ freeSpace = fetchConstantFreeSpace(regexResult.at(1), lc);
+ goto found;
+ }
+ throw exception::Exception("In DiskSystemFreeSpaceList::fetchDiskSystemFreeSpace(): could not interpret free space query URL.");
+ found:
+ DiskSystemFreeSpace & entry = operator [](ds);
+ entry.freeSpace = freeSpace;
+ entry.fetchTime = ::time(nullptr);
+ entry.targetedFreeSpace = m_systemList.at(ds).targetedFreeSpace;
+ }
+}
+
+//------------------------------------------------------------------------------
+// DiskSystemFreeSpaceList::fetchFileSystemFreeSpace()
+//------------------------------------------------------------------------------
+uint64_t DiskSystemFreeSpaceList::fetchEosFreeSpace(const std::string& instanceAddress, log::LogContext & lc) {
+ threading::SubProcess sp("/usr/bin/eos", {"/usr/bin/eos", std::string("root://")+instanceAddress, "space", "ls", "-m"});
+ sp.wait();
+ try {
+ exception::Errnum::throwOnNonZero(sp.exitValue(),
+ std::string("In DiskSystemFreeSpaceList::fetchEosFreeSpace(), failed to call \"eos root://") +
+ instanceAddress + " space ls -m\"");
+ } catch (exception::Exception & ex) {
+ ex.getMessage() << " instanceAddress: " << instanceAddress << " stderr: " << sp.stderr();
+ throw;
+ }
+ if (sp.wasKilled()) {
+ exception::Exception ex("In DiskSystemFreeSpaceList::fetchEosFreeSpace(): eos space ls -m killed by signal: ");
+ ex.getMessage() << utils::toString(sp.killSignal());
+ throw ex;
+ }
+ // Look for the result line for default space.
+ std::istringstream spStdoutIss(sp.stdout());
+ std::string defaultSpaceLine;
+ utils::Regex defaultSpaceRe("^.*name=default .*$");
+ do {
+ std::string spStdoutLine;
+ std::getline(spStdoutIss, spStdoutLine);
+ auto res = defaultSpaceRe.exec(spStdoutLine);
+ if (res.size()) {
+ defaultSpaceLine = res.at(0);
+ goto defaultFound;
+ }
+ } while (!spStdoutIss.eof());
+ throw exception::Exception("In DiskSystemFreeSpaceList::fetchEosFreeSpace(): could not find line for default space.");
+
+defaultFound:
+ // Look for the parameters in the result line.
+ utils::Regex rwSpaceRegex("sum.stat.statfs.capacity\\?configstatus@rw=([0-9]+) ");
+ auto rwSpaceRes = rwSpaceRegex.exec(defaultSpaceLine);
+ if (rwSpaceRes.empty())
+ throw exception::Exception(
+ "In DiskSystemFreeSpaceList::fetchEosFreeSpace(): failed to parse parameter sum.stat.statfs.capacity?configstatus@rw.");
+ utils::Regex usedSpaceRegex("sum.stat.statfs.usedbytes=([0-9]+) ");
+ auto usedSpaceRes = usedSpaceRegex.exec(sp.stdout());
+ if (usedSpaceRes.empty())
+ throw exception::Exception("In DiskSystemFreeSpaceList::fetchEosFreeSpace(): failed to parse parameter sum.stat.statfs.usedbytes.");
+ return utils::toUint64(rwSpaceRes.at(1)) - utils::toUint64(usedSpaceRes.at(1));
+}
+
+//------------------------------------------------------------------------------
+// DiskSystemFreeSpaceList::fetchFileSystemFreeSpace()
+//------------------------------------------------------------------------------
+uint64_t DiskSystemFreeSpaceList::fetchConstantFreeSpace(const std::string& instanceAddress, log::LogContext & lc) {
+ return utils::toUint64(instanceAddress);
+}
+
+
+
+}} // namespace cta::disk
\ No newline at end of file
diff --git a/disk/DiskSystem.hpp b/disk/DiskSystem.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..72afa78992ae715fa33c9b83b3e39d85f7d5678c
--- /dev/null
+++ b/disk/DiskSystem.hpp
@@ -0,0 +1,91 @@
+/*
+ * The CERN Tape Archive (CTA) project
+ * Copyright (C) 2015 CERN
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include "common/utils/Regex.hpp"
+#include "common/dataStructures/EntryLog.hpp"
+#include "common/log/LogContext.hpp"
+#include <string>
+#include <list>
+#include <set>
+
+namespace cta { namespace disk {
+
+/**
+ * Description of a disk system as defined by operators.
+ * Defines:
+ * - a name used an index
+ * - a regular expression allowing matching destination URLs for this disk system
+ * - a query URL that describes a method to query the free space from the filesystem
+ * - a refresh interval (seconds) defining how long do we use a
+ * - a targeted free space (margin) based on the free space update latency (inherent to the file system and induced by the refresh
+ * interval), and the expected external bandwidth from sources external to CTA.
+ */
+struct DiskSystem {
+ std::string name;
+ std::string fileRegexp;
+ std::string freeSpaceQueryURL;
+ uint64_t refreshInterval;
+ uint64_t targetedFreeSpace;
+ uint64_t sleepTime;
+ cta::common::dataStructures::EntryLog creationLog;
+ cta::common::dataStructures::EntryLog lastModificationLog;
+ std::string comment;
+};
+
+class DiskSystemList: public std::list<DiskSystem> {
+ using std::list<DiskSystem>::list;
+
+public:
+ /** Get the filesystem for a given destination URL */
+ std::string getDSNAme(const std::string &fileURL) const;
+
+ /** Get the file system parameters from a file system name */
+ const DiskSystem & at(const std::string &name) const;
+
+private:
+ struct PointerAndRegex {
+ PointerAndRegex(const DiskSystem & dsys, const std::string &re): ds(dsys), regex(re) {}
+ const DiskSystem & ds;
+ utils::Regex regex;
+ };
+
+ mutable std::list<PointerAndRegex> m_pointersAndRegexes;
+
+};
+
+struct DiskSystemFreeSpace {
+ uint64_t freeSpace;
+ uint64_t targetedFreeSpace;
+ time_t fetchTime;
+};
+
+class DiskSystemFreeSpaceList: public std::map<std::string, DiskSystemFreeSpace> {
+public:
+ DiskSystemFreeSpaceList(DiskSystemList &diskSystemList): m_systemList(diskSystemList) {}
+ void fetchDiskSystemFreeSpace(const std::set<std::string> &diskSystems, log::LogContext & lc);
+ const DiskSystemList &getDiskSystemList() { return m_systemList; }
+private:
+ DiskSystemList &m_systemList;
+ uint64_t fetchEosFreeSpace(const std::string & instanceAddress, log::LogContext & lc);
+ uint64_t fetchConstantFreeSpace(const std::string & instanceAddress, log::LogContext & lc);
+};
+
+}} // namespace cta::common
+
diff --git a/objectstore/Agent.cpp b/objectstore/Agent.cpp
index 89cbe6154614afa3925aec59a431a8241b071c7d..4e6c3e91ef2905ba2af588bfa448e79245d13817 100644
--- a/objectstore/Agent.cpp
+++ b/objectstore/Agent.cpp
@@ -45,6 +45,7 @@ void cta::objectstore::Agent::initialize() {
m_payload.set_timeout_us(120*1000*1000);
m_payload.set_description("");
m_payload.set_being_garbage_collected(false);
+ m_payload.set_gc_needed(false);
m_payloadInterpreted = true;
}
@@ -149,6 +150,17 @@ void cta::objectstore::Agent::setBeingGarbageCollected() {
m_payload.set_being_garbage_collected(true);
}
+void cta::objectstore::Agent::setNeedsGarbageCollection() {
+ checkPayloadWritable();
+ m_payload.set_gc_needed(true);
+}
+
+bool cta::objectstore::Agent::needsGarbageCollection() {
+ checkPayloadReadable();
+ if (!m_payload.has_gc_needed()) return false;
+ return m_payload.gc_needed();
+}
+
void cta::objectstore::Agent::garbageCollect(const std::string& presumedOwner, AgentReference & agentReference, log::LogContext & lc,
cta::catalogue::Catalogue & catalogue) {
checkPayloadWritable();
diff --git a/objectstore/Agent.hpp b/objectstore/Agent.hpp
index da25916118e05e87baab7a875eb2f7a46239b94a..de371d1134ab34c6db3dafb780011164a185f627 100644
--- a/objectstore/Agent.hpp
+++ b/objectstore/Agent.hpp
@@ -62,6 +62,10 @@ public:
bool isBeingGarbageCollected();
+ void setNeedsGarbageCollection();
+
+ bool needsGarbageCollection();
+
void garbageCollect(const std::string &presumedOwner, AgentReference & agentReference, log::LogContext & lc,
cta::catalogue::Catalogue & catalogue) override;
diff --git a/objectstore/AgentWatchdog.hpp b/objectstore/AgentWatchdog.hpp
index 9f1a157a1913742a15996eeb6c34d488ea7a0a53..0ae9af672bb34e8dd8980078e158c48cd70b43ab 100644
--- a/objectstore/AgentWatchdog.hpp
+++ b/objectstore/AgentWatchdog.hpp
@@ -25,25 +25,28 @@ namespace cta { namespace objectstore {
class AgentWatchdog {
public:
AgentWatchdog(const std::string & name, Backend & os): m_agent(name, os),
- m_heartbeatCounter(readHeartbeat()) {
+ m_heartbeatCounter(readGCData().heartbeat) {
m_agent.fetchNoLock();
m_timeout = m_agent.getTimeout();
}
bool checkAlive() {
- uint64_t newHeartBeatCount;
- try {
- newHeartBeatCount = readHeartbeat();
+ struct gcData newGCData;
+ try {
+ newGCData = readGCData();
} catch (Backend::NoSuchObject &) {
// The agent could be gone. This is not an error. Mark it as alive,
// and will be trimmed later.
return true;
}
auto timer = m_timer.secs();
- if (newHeartBeatCount == m_heartbeatCounter && timer > m_timeout)
+ // If GC is required, it's easy...
+ if (newGCData.needsGC) return false;
+ // If heartbeat has not moved for more than the timeout, we declare the agent dead.
+ if (newGCData.heartbeat == m_heartbeatCounter && timer > m_timeout)
return false;
- if (newHeartBeatCount != m_heartbeatCounter) {
- m_heartbeatCounter = newHeartBeatCount;
+ if (newGCData.heartbeat != m_heartbeatCounter) {
+ m_heartbeatCounter = newGCData.heartbeat;
m_timer.reset();
}
return true;
@@ -51,7 +54,9 @@ public:
std::list<log::Param> getDeadAgentDetails() {
std::list<log::Param> ret;
- ret.push_back(log::Param("currentHeartbeat", readHeartbeat()));
+ auto gcData = readGCData();
+ ret.push_back(log::Param("currentHeartbeat", gcData.heartbeat));
+ ret.push_back(log::Param("GCRequested", gcData.needsGC?"true":"false"));
ret.push_back(log::Param("timeout", m_timeout));
ret.push_back(log::Param("timer", m_timer.secs()));
ret.push_back(log::Param("heartbeatAtTimerStart", m_heartbeatCounter));
@@ -72,9 +77,16 @@ private:
uint64_t m_heartbeatCounter;
double m_timeout;
- uint64_t readHeartbeat() {
+ struct gcData {
+ uint64_t heartbeat;
+ bool needsGC;
+ };
+ struct gcData readGCData() {
m_agent.fetchNoLock();
- return m_agent.getHeartbeatCount();
+ struct gcData ret;
+ ret.heartbeat = m_agent.getHeartbeatCount();
+ ret.needsGC = m_agent.needsGarbageCollection();
+ return ret;
}
};
diff --git a/objectstore/AlgorithmsTest.cpp b/objectstore/AlgorithmsTest.cpp
index a4efb61965faf26aa9a942ef44297fd4aa51987b..9505536bc402b3cd90e99c26cf2f1c5182eff82e 100644
--- a/objectstore/AlgorithmsTest.cpp
+++ b/objectstore/AlgorithmsTest.cpp
@@ -35,7 +35,7 @@
namespace unitTests {
void fillRetrieveRequests(
- typename cta::objectstore::ContainerAlgorithms<cta::objectstore::RetrieveQueue,cta::objectstore::RetrieveQueueToTransferForUser>::InsertedElement::list &requests,
+ typename cta::objectstore::ContainerAlgorithms<cta::objectstore::RetrieveQueue,cta::objectstore::RetrieveQueueToTransfer>::InsertedElement::list &requests,
std::list<std::unique_ptr<cta::objectstore::RetrieveRequest> >& requestPtrs, //List to avoid memory leak on ArchiveQueueAlgorithms test
cta::objectstore::BackendVFS &be,
cta::objectstore::AgentReference &agentRef)
@@ -75,8 +75,8 @@ void fillRetrieveRequests(
rqc.mountPolicy.retrieveMinRequestAge = 1;
rqc.mountPolicy.retrievePriority = 1;
requestPtrs.emplace_back(new cta::objectstore::RetrieveRequest(rrAddr, be));
- requests.emplace_back(ContainerAlgorithms<RetrieveQueue,RetrieveQueueToTransferForUser>::InsertedElement{
- requestPtrs.back().get(), 1, i, 667, mp, serializers::RetrieveJobStatus::RJS_ToTransferForUser, cta::nullopt
+ requests.emplace_back(ContainerAlgorithms<RetrieveQueue,RetrieveQueueToTransfer>::InsertedElement{
+ requestPtrs.back().get(), 1, i, 667, mp, serializers::RetrieveJobStatus::RJS_ToTransfer, cta::nullopt, cta::nullopt
});
auto &rr = *requests.back().retrieveRequest;
rr.initialize();
@@ -189,7 +189,7 @@ TEST(ObjectStore, RetrieveQueueAlgorithms) {
agent.initialize();
agent.insertAndRegisterSelf(lc);
std::list<std::unique_ptr<RetrieveRequest> > requestsPtrs;
- ContainerAlgorithms<RetrieveQueue,RetrieveQueueToTransferForUser>::InsertedElement::list requests;
+ ContainerAlgorithms<RetrieveQueue,RetrieveQueueToTransfer>::InsertedElement::list requests;
fillRetrieveRequests(requests, requestsPtrs, be, agentRef); //memory leak here
{
@@ -208,18 +208,18 @@ TEST(ObjectStore, RetrieveQueueAlgorithms) {
rel2.release();
agent2.initialize();
agent2.insertAndRegisterSelf(lc);
- ContainerAlgorithms<RetrieveQueue,RetrieveQueueToTransferForUser>::InsertedElement::list requests2;
+ ContainerAlgorithms<RetrieveQueue,RetrieveQueueToTransfer>::InsertedElement::list requests2;
std::list<std::unique_ptr<RetrieveRequest> > requestsPtrs2;
fillRetrieveRequests(requests2, requestsPtrs2,be2, agentRef2);
auto a1 = agentRef2.getAgentAddress();
auto a2 = agentRef2.getAgentAddress();
- ContainerAlgorithms<RetrieveQueue,RetrieveQueueToTransferForUser> retrieveAlgos2(be2, agentRef2);
+ ContainerAlgorithms<RetrieveQueue,RetrieveQueueToTransfer> retrieveAlgos2(be2, agentRef2);
retrieveAlgos2.referenceAndSwitchOwnershipIfNecessary("VID",
a2, a1, requests2, lc);
}
- ContainerAlgorithms<RetrieveQueue,RetrieveQueueToTransferForUser> retrieveAlgos(be, agentRef);
+ ContainerAlgorithms<RetrieveQueue,RetrieveQueueToTransfer> retrieveAlgos(be, agentRef);
try {
ASSERT_EQ(requests.size(), 10);
@@ -227,19 +227,19 @@ TEST(ObjectStore, RetrieveQueueAlgorithms) {
agentRef.getAgentAddress(), requests, lc);
// Now get the requests back
- ContainerTraits<RetrieveQueue,RetrieveQueueToTransferForUser>::PopCriteria popCriteria;
+ ContainerTraits<RetrieveQueue,RetrieveQueueToTransfer>::PopCriteria popCriteria;
popCriteria.bytes = std::numeric_limits<decltype(popCriteria.bytes)>::max();
popCriteria.files = 100;
auto poppedJobs = retrieveAlgos.popNextBatch("VID", popCriteria, lc);
ASSERT_EQ(poppedJobs.summary.files, 10);
// Validate that the summary has the same information as the popped elements
- ContainerTraits<RetrieveQueue,RetrieveQueueToTransferForUser>::PoppedElementsSummary s;
+ ContainerTraits<RetrieveQueue,RetrieveQueueToTransfer>::PoppedElementsSummary s;
for(auto &e: poppedJobs.elements) {
- s += ContainerTraits<RetrieveQueue,RetrieveQueueToTransferForUser>::getElementSummary(e);
+ s += ContainerTraits<RetrieveQueue,RetrieveQueueToTransfer>::getElementSummary(e);
}
ASSERT_EQ(s, poppedJobs.summary);
- } catch (ContainerTraits<RetrieveQueue,RetrieveQueueToTransferForUser>::OwnershipSwitchFailure & ex) {
+ } catch (ContainerTraits<RetrieveQueue,RetrieveQueueToTransfer>::OwnershipSwitchFailure & ex) {
for (auto & e: ex.failedElements) {
try {
throw e.failure;
diff --git a/objectstore/BackendPopulator.cpp b/objectstore/BackendPopulator.cpp
index 3a7f71a1af683f77cd2ca404b42be1b19901d246..7c6d84ceb5e74c0ce857acfedf5dc881baefe769 100644
--- a/objectstore/BackendPopulator.cpp
+++ b/objectstore/BackendPopulator.cpp
@@ -66,6 +66,8 @@ BackendPopulator::~BackendPopulator() throw() {
agent.fetch();
if (m_leaveNonEmptyAgentBehind && !agent.isEmpty()) {
cta::log::ScopedParamContainer params(m_lc);
+ agent.setNeedsGarbageCollection();
+ agent.commit();
params.add("agentObject", agent.getAddressIfSet())
.add("ownedObjectCount", agent.getOwnershipList().size());
m_lc.log(log::WARNING, "In BackendPopulator::~BackendPopulator(): not deleting non-empty agent object, left for garbage collection.");
diff --git a/objectstore/CMakeLists.txt b/objectstore/CMakeLists.txt
index d30eb6aa7f687da85aebd42283ea7f34644a7115..60e01f8ff911ff290aa55f2d4723e64c909b6f89 100644
--- a/objectstore/CMakeLists.txt
+++ b/objectstore/CMakeLists.txt
@@ -80,7 +80,6 @@ add_library (ctaobjectstore SHARED
RetrieveQueueFailedAlgorithms.cpp
RetrieveQueueToReportToRepackForSuccessAlgorithms.cpp
RetrieveQueueToReportToRepackForFailureAlgorithms.cpp
- RetrieveQueueToTransferForRepackAlgorithms.cpp
JobQueueType.cpp
Sorter.cpp
ArchiveRequest.cpp
diff --git a/objectstore/DriveState.cpp b/objectstore/DriveState.cpp
index df47cb6caaaa0f3f159bf7f2b9019d02d1d8e0c1..b175f8951675e2577b6f65023337a572a11a1890 100644
--- a/objectstore/DriveState.cpp
+++ b/objectstore/DriveState.cpp
@@ -268,11 +268,78 @@ void DriveState::setConfig(const cta::tape::daemon::TapedConfiguration& tapedCon
fillConfig(config->fileCatalogConfigFile);
}
+//------------------------------------------------------------------------------
+// DriveState::setTpConfig())
+//------------------------------------------------------------------------------
void DriveState::setTpConfig(const cta::tape::daemon::TpconfigLine& configLine){
m_payload.set_dev_file_name(configLine.devFilename);
m_payload.set_raw_library_slot(configLine.rawLibrarySlot);
}
+//------------------------------------------------------------------------------
+// DriveState::getDiskSpaceReservations())
+//------------------------------------------------------------------------------
+std::map<std::string, uint64_t> DriveState::getDiskSpaceReservations() {
+ checkHeaderReadable();
+ std::map<std::string, uint64_t> ret;
+ for (auto &dsr: m_payload.disk_space_reservations()) {
+ ret[dsr.disk_system_name()] = dsr.reserved_bytes();
+ }
+ return ret;
+}
+
+//------------------------------------------------------------------------------
+// DriveState::addDiskSpaceReservation())
+//------------------------------------------------------------------------------
+void DriveState::addDiskSpaceReservation(const std::string& diskSystemName, uint64_t bytes) {
+ checkPayloadWritable();
+ for (auto dsr: *m_payload.mutable_disk_space_reservations()) {
+ if (dsr.disk_system_name() == diskSystemName) {
+ dsr.set_reserved_bytes(dsr.reserved_bytes() + bytes);
+ return;
+ }
+ }
+ auto * newDsr = m_payload.mutable_disk_space_reservations()->Add();
+ newDsr->set_disk_system_name(diskSystemName);
+ newDsr->set_reserved_bytes(bytes);
+}
+
+//------------------------------------------------------------------------------
+// DriveState::substractDiskSpaceReservation())
+//------------------------------------------------------------------------------
+void DriveState::substractDiskSpaceReservation(const std::string& diskSystemName, uint64_t bytes) {
+ checkPayloadWritable();
+ size_t index=0;
+ for (auto dsr: *m_payload.mutable_disk_space_reservations()) {
+ if (dsr.disk_system_name() == diskSystemName) {
+ if (bytes > dsr.reserved_bytes())
+ throw NegativeDiskSpaceReservationReached(
+ "In DriveState::substractDiskSpaceReservation(): we would reach a negative reservation size.");
+ dsr.set_reserved_bytes(dsr.reserved_bytes() - bytes);
+ if (!dsr.reserved_bytes()) {
+ // We can remove this entry from the list.
+ auto * mdsr = m_payload.mutable_disk_space_reservations();
+ mdsr->SwapElements(index, mdsr->size()-1);
+ mdsr->RemoveLast();
+ }
+ return;
+ } else {
+ ++index;
+ }
+ }
+ if (bytes)
+ throw NegativeDiskSpaceReservationReached(
+ "In DriveState::substractDiskSpaceReservation(): Trying to substract bytes without previous reservation.");
+}
+
+//------------------------------------------------------------------------------
+// DriveState::resetDiskSpaceReservation())
+//------------------------------------------------------------------------------
+void DriveState::resetDiskSpaceReservation() {
+ checkPayloadWritable();
+ m_payload.mutable_disk_space_reservations()->Clear();
+}
+
//------------------------------------------------------------------------------
// DriveState::dump())
//------------------------------------------------------------------------------
@@ -286,10 +353,13 @@ std::string DriveState::dump() {
return headerDump;
}
+//------------------------------------------------------------------------------
+// DriveState::commit()
+//------------------------------------------------------------------------------
void DriveState::commit(){
checkPayloadWritable();
m_payload.set_cta_version(CTA_VERSION);
ObjectOps<serializers::DriveState, serializers::DriveState_t>::commit();
}
-}} // namespace cta::objectstore
\ No newline at end of file
+}} // namespace cta::objectstore
diff --git a/objectstore/DriveState.hpp b/objectstore/DriveState.hpp
index 636622761fd096f2c0648e262cdd846517d9ae7f..ab405df93a61b9cfc812bfd543f6fdd18ce9c1e0 100644
--- a/objectstore/DriveState.hpp
+++ b/objectstore/DriveState.hpp
@@ -46,6 +46,13 @@ public:
// Data access
cta::common::dataStructures::DriveState getState();
void setState(cta::common::dataStructures::DriveState & state);
+
+ std::map<std::string, uint64_t> getDiskSpaceReservations();
+ void addDiskSpaceReservation(const std::string & diskSystemName, uint64_t bytes);
+ CTA_GENERATE_EXCEPTION_CLASS(NegativeDiskSpaceReservationReached);
+ void substractDiskSpaceReservation(const std::string & diskSystemName, uint64_t bytes);
+ void resetDiskSpaceReservation();
+
void setConfig(const cta::tape::daemon::TapedConfiguration &tapedConfiguration);
void setTpConfig(const cta::tape::daemon::TpconfigLine &tpConfigLine);
/**
diff --git a/objectstore/GarbageCollector.cpp b/objectstore/GarbageCollector.cpp
index 89d7c4a9546cbbd2ae8ed2241a447c79f1d4de8c..052f4bb2098d62aec6c2664243bc89c31c6499da 100644
--- a/objectstore/GarbageCollector.cpp
+++ b/objectstore/GarbageCollector.cpp
@@ -338,7 +338,7 @@ void GarbageCollector::OwnedObjectSorter::sortFetchedObjects(Agent& agent, std::
std::set<std::string> candidateVids;
bool disabledTape = rr->getRepackInfo().forceDisabledTape;
for (auto & j: rr->dumpJobs()) {
- if(j.status==RetrieveJobStatus::RJS_ToTransferForUser) {
+ if(j.status==RetrieveJobStatus::RJS_ToTransfer) {
for (auto &tf: rr->getArchiveFile().tapeFiles) {
if ((tf.copyNb == j.copyNb) && (tf.supersededByVid.empty()))
candidateVids.insert(tf.vid);
@@ -615,7 +615,7 @@ void GarbageCollector::OwnedObjectSorter::lockFetchAndUpdateRetrieveJobs(Agent&
for (auto &tf: rr->getArchiveFile().tapeFiles) {
if (tf.vid == vid) {
jta.push_back({tf.copyNb, tf.fSeq, rr->getAddressIfSet(), rr->getArchiveFile().fileSize,
- rr->getRetrieveFileQueueCriteria().mountPolicy, rr->getEntryLog().time, rr->getActivity()});
+ rr->getRetrieveFileQueueCriteria().mountPolicy, rr->getEntryLog().time, rr->getActivity(), rr->getDiskSystemName()});
}
}
}
diff --git a/objectstore/GarbageCollectorTest.cpp b/objectstore/GarbageCollectorTest.cpp
index 1d3731caa12a291e0669b5aad0862cc1bcbf7afa..e9ff8227f4bd415f4d4068ea0e4f368b7b83b7cb 100644
--- a/objectstore/GarbageCollectorTest.cpp
+++ b/objectstore/GarbageCollectorTest.cpp
@@ -603,7 +603,8 @@ TEST(ObjectStore, GarbageCollectorRetrieveRequest) {
cta::objectstore::ScopedExclusiveLock rql(rq);
rq.fetch();
std::list <cta::objectstore::RetrieveQueue::JobToAdd> jta;
- jta.push_back({1,rqc.archiveFile.tapeFiles.front().fSeq, rr.getAddressIfSet(), rqc.archiveFile.fileSize, rqc.mountPolicy, sReq.creationLog.time, cta::nullopt});
+ jta.push_back({1,rqc.archiveFile.tapeFiles.front().fSeq, rr.getAddressIfSet(), rqc.archiveFile.fileSize, rqc.mountPolicy,
+ sReq.creationLog.time, cta::nullopt, cta::nullopt});
rq.addJobsAndCommit(jta, agentRef, lc);
}
if (pass < 5) { pass++; continue; }
@@ -1163,7 +1164,7 @@ TEST(ObjectStore, GarbageCollectorRetrieveAllStatusesAndQueues) {
sReq.archiveFileID = rqc.archiveFile.archiveFileID;
sReq.creationLog.time=time(nullptr);
rr.setSchedulerRequest(sReq);
- rr.setJobStatus(2,cta::objectstore::serializers::RetrieveJobStatus::RJS_ToTransferForUser);
+ rr.setJobStatus(2,cta::objectstore::serializers::RetrieveJobStatus::RJS_ToTransfer);
rr.setOwner(agentToTransferForUser.getAddressIfSet());
rr.setActiveCopyNumber(0);
rr.insert();
@@ -1632,7 +1633,7 @@ TEST(ObjectStore, GarbageCollectorRetrieveRequestRepackDisabledTape){
sReq.archiveFileID = rqc.archiveFile.archiveFileID;
sReq.creationLog.time=time(nullptr);
rr.setSchedulerRequest(sReq);
- rr.setJobStatus(2,cta::objectstore::serializers::RetrieveJobStatus::RJS_ToTransferForUser);
+ rr.setJobStatus(2,cta::objectstore::serializers::RetrieveJobStatus::RJS_ToTransfer);
rr.setOwner(agentToTransferForUser.getAddressIfSet());
rr.setActiveCopyNumber(0);
@@ -1693,7 +1694,7 @@ TEST(ObjectStore, GarbageCollectorRetrieveRequestRepackDisabledTape){
cta::objectstore::ScopedExclusiveLock sel(rr);
rr.fetch();
rr.setOwner(agentRefToTransferDisabledTapeAutoGc.getAgentAddress());
- rr.setJobStatus(2,cta::objectstore::serializers::RetrieveJobStatus::RJS_ToTransferForUser);
+ rr.setJobStatus(2,cta::objectstore::serializers::RetrieveJobStatus::RJS_ToTransfer);
rr.commit();
agentRefToTransferDisabledTapeAutoGc.addToOwnership(rr.getAddressIfSet(),be);
diff --git a/objectstore/JobQueueType.hpp b/objectstore/JobQueueType.hpp
index 4280616e4d1b6013fe09d3e98b699559d7ed0785..2557f6da5d75cfac3f57eb0644936a1051059150 100644
--- a/objectstore/JobQueueType.hpp
+++ b/objectstore/JobQueueType.hpp
@@ -21,6 +21,13 @@
#include <string>
namespace cta { namespace objectstore {
-enum class JobQueueType { JobsToTransferForUser, FailedJobs, JobsToReportToUser, JobsToReportToRepackForSuccess, JobsToReportToRepackForFailure, JobsToTransferForRepack };
+enum class JobQueueType {
+ JobsToTransferForUser,
+ FailedJobs,
+ JobsToReportToUser,
+ JobsToReportToRepackForSuccess,
+ JobsToReportToRepackForFailure,
+ JobsToTransferForRepack
+};
std::string toString(JobQueueType queueType);
}} // namespace cta::objectstore
\ No newline at end of file
diff --git a/objectstore/ObjectOps.hpp b/objectstore/ObjectOps.hpp
index aceb9d623037c760c513ebb87833858bf8301ec3..dfa757b2617f5d4bcdcc4376e2a9f39d77200e6e 100644
--- a/objectstore/ObjectOps.hpp
+++ b/objectstore/ObjectOps.hpp
@@ -43,7 +43,7 @@ struct ArchiveQueueToReportToRepackForSuccess;
struct ArchiveQueueFailed;
struct ArchiveQueueToTransferForRepack;
struct RetrieveQueue;
-struct RetrieveQueueToTransferForUser;
+struct RetrieveQueueToTransfer;
struct RetrieveQueueToReportForUser;
struct RetrieveQueueFailed;
struct RetrieveQueueToReportToRepackForSuccess;
@@ -64,7 +64,7 @@ class ObjectOpsBase {
friend ContainerTraits<ArchiveQueue,ArchiveQueueToTransferForRepack>;
friend ContainerTraits<ArchiveQueue,ArchiveQueueToReportToRepackForFailure>;
friend ContainerTraits<ArchiveQueue,ArchiveQueueToReportToRepackForSuccess>;
- friend ContainerTraits<RetrieveQueue,RetrieveQueueToTransferForUser>;
+ friend ContainerTraits<RetrieveQueue,RetrieveQueueToTransfer>;
friend ContainerTraits<RetrieveQueue,RetrieveQueueToReportForUser>;
friend ContainerTraits<RetrieveQueue,RetrieveQueueFailed>;
friend ContainerTraits<RetrieveQueue,RetrieveQueueToReportToRepackForSuccess>;
diff --git a/objectstore/RepackRequest.cpp b/objectstore/RepackRequest.cpp
index 2ce5898f2c638562d1d09112e2e16165938a867a..a766a86f2cc367be61ff2905bfc5499254c77721 100644
--- a/objectstore/RepackRequest.cpp
+++ b/objectstore/RepackRequest.cpp
@@ -328,26 +328,41 @@ uint64_t RepackRequest::getLastExpandedFSeq() {
return m_payload.lastexpandedfseq();
}
+//------------------------------------------------------------------------------
+// RepackRequest::setTotalFileToRetrieve()
+//------------------------------------------------------------------------------
void RepackRequest::setTotalFileToRetrieve(const uint64_t nbFilesToRetrieve){
checkPayloadWritable();
m_payload.set_totalfilestoretrieve(nbFilesToRetrieve);
}
+//------------------------------------------------------------------------------
+// RepackRequest::setTotalBytesToRetrieve()
+//------------------------------------------------------------------------------
void RepackRequest::setTotalBytesToRetrieve(const uint64_t nbBytesToRetrieve){
checkPayloadWritable();
m_payload.set_totalbytestoretrieve(nbBytesToRetrieve);
}
+//------------------------------------------------------------------------------
+// RepackRequest::setTotalFileToArchive()
+//------------------------------------------------------------------------------
void RepackRequest::setTotalFileToArchive(const uint64_t nbFilesToArchive){
checkPayloadWritable();
m_payload.set_totalfilestoarchive(nbFilesToArchive);
}
+//------------------------------------------------------------------------------
+// RepackRequest::setTotalBytesToArchive()
+//------------------------------------------------------------------------------
void RepackRequest::setTotalBytesToArchive(const uint64_t nbBytesToArchive) {
checkPayloadWritable();
m_payload.set_totalbytestoarchive(nbBytesToArchive);
- }
+}
+//------------------------------------------------------------------------------
+// RepackRequest::getTotalStatsFile()
+//------------------------------------------------------------------------------
cta::SchedulerDatabase::RepackRequest::TotalStatsFiles RepackRequest::getTotalStatsFile() {
checkPayloadReadable();
cta::SchedulerDatabase::RepackRequest::TotalStatsFiles ret;
diff --git a/objectstore/RetrieveQueue.cpp b/objectstore/RetrieveQueue.cpp
index 755e48550338480eb412a9098909741e884246d1..e6239c1472c51156193e30e2437af76726ca7cf7 100644
--- a/objectstore/RetrieveQueue.cpp
+++ b/objectstore/RetrieveQueue.cpp
@@ -214,6 +214,19 @@ std::string RetrieveQueue::getVid() {
return m_payload.vid();
}
+void RetrieveQueue::resetSleepForFreeSpaceStartTime() {
+ checkPayloadWritable();
+ m_payload.clear_sleep_for_free_space_since();
+ m_payload.clear_disk_system_slept_for();
+}
+
+void RetrieveQueue::setSleepForFreeSpaceStartTimeAndName(time_t time, const std::string & diskSystemName, uint64_t sleepTime) {
+ checkPayloadWritable();
+ m_payload.set_sleep_for_free_space_since((uint64_t)time);
+ m_payload.set_disk_system_slept_for(diskSystemName);
+ m_payload.set_sleep_time(sleepTime);
+}
+
std::string RetrieveQueue::dump() {
checkPayloadReadable();
google::protobuf::util::JsonPrintOptions options;
@@ -531,7 +544,7 @@ auto RetrieveQueue::addJobsIfNecessaryAndCommit(std::list<JobToAdd> & jobsToAdd,
}
shardsDumps.emplace_back(std::list<JobDump>());
for (auto & j: s->dumpJobs()) {
- shardsDumps.back().emplace_back(JobDump({j.address, j.copyNb, j.size}));
+ shardsDumps.back().emplace_back(JobDump({j.address, j.copyNb, j.size, j.activityDescription, j.diskSystemName}));
}
nextShard:
s++;
@@ -577,6 +590,13 @@ RetrieveQueue::JobsSummary RetrieveQueue::getJobsSummary() {
for (auto ra: retrieveActivityCountMap.getActivities(ret.priority)) {
ret.activityCounts.push_back({ra.diskInstanceName, ra.activity, ra.weight, ra.count});
}
+ if (m_payload.has_sleep_for_free_space_since()) {
+ JobsSummary::SleepInfo si;
+ si.diskSystemSleptFor = m_payload.disk_system_slept_for();
+ si.sleepStartTime = m_payload.sleep_for_free_space_since();
+ si.sleepTime = m_payload.sleep_time();
+ ret.sleepInfo = si;
+ }
} else {
ret.maxDrivesAllowed = 0;
ret.priority = 0;
@@ -606,7 +626,7 @@ auto RetrieveQueue::dumpJobs() -> std::list<JobDump> {
goto nextShard;
}
for (auto & j: s->dumpJobs()) {
- ret.emplace_back(JobDump{j.address, j.copyNb, j.size});
+ ret.emplace_back(JobDump{j.address, j.copyNb, j.size, j.activityDescription, j.diskSystemName});
}
nextShard:
s++; sf++;
@@ -614,7 +634,7 @@ auto RetrieveQueue::dumpJobs() -> std::list<JobDump> {
return ret;
}
-auto RetrieveQueue::getCandidateList(uint64_t maxBytes, uint64_t maxFiles, std::set<std::string> retrieveRequestsToSkip) -> CandidateJobList {
+auto RetrieveQueue::getCandidateList(uint64_t maxBytes, uint64_t maxFiles, const std::set<std::string> & retrieveRequestsToSkip, const std::set<std::string> & diskSystemsToSkip) -> CandidateJobList {
checkPayloadReadable();
CandidateJobList ret;
for(auto & rqsp: m_payload.retrievequeueshards()) {
@@ -623,7 +643,8 @@ auto RetrieveQueue::getCandidateList(uint64_t maxBytes, uint64_t maxFiles, std::
// Fetch the shard
RetrieveQueueShard rqs(rqsp.address(), m_objectStore);
rqs.fetchNoLock();
- auto shardCandidates = rqs.getCandidateJobList(maxBytes - ret.candidateBytes, maxFiles - ret.candidateFiles, retrieveRequestsToSkip);
+ auto shardCandidates = rqs.getCandidateJobList(maxBytes - ret.candidateBytes, maxFiles - ret.candidateFiles,
+ retrieveRequestsToSkip, diskSystemsToSkip);
ret.candidateBytes += shardCandidates.candidateBytes;
ret.candidateFiles += shardCandidates.candidateFiles;
// We overwrite the remaining values each time as the previous
diff --git a/objectstore/RetrieveQueue.hpp b/objectstore/RetrieveQueue.hpp
index d9566754e42c0a23d05491452a6470d247d37674..9120d1915b5c380f3e9b397b465fe509a01eadb0 100644
--- a/objectstore/RetrieveQueue.hpp
+++ b/objectstore/RetrieveQueue.hpp
@@ -66,7 +66,8 @@ public:
uint64_t fileSize;
cta::common::dataStructures::MountPolicy policy;
time_t startTime;
- optional<RetrieveActivityDescription> activityDescription;
+ optional<RetrieveActivityDescription> activityDescription;
+ optional<std::string> diskSystemName;
};
void addJobsAndCommit(std::list<JobToAdd> & jobsToAdd, AgentReference & agentReference, log::LogContext & lc);
// This version will check for existence of the job in the queue before
@@ -91,12 +92,24 @@ public:
uint64_t count;
};
std::list<ActivityCount> activityCounts;
+ struct SleepInfo {
+ time_t sleepStartTime;
+ std::string diskSystemSleptFor;
+ uint64_t sleepTime;
+ };
+ optional<SleepInfo> sleepInfo;
};
JobsSummary getJobsSummary();
struct JobDump {
std::string address;
uint32_t copyNb;
uint64_t size;
+ struct ActivityDescription {
+ std::string diskInstanceName;
+ std::string activity;
+ };
+ optional<ActivityDescription> activity;
+ optional<std::string> diskSystemName;
};
std::list<JobDump> dumpJobs();
struct CandidateJobList {
@@ -108,7 +121,9 @@ public:
};
// The set of retrieve requests to skip are requests previously identified by the caller as bad,
// which still should be removed from the queue. They will be disregarded from listing.
- CandidateJobList getCandidateList(uint64_t maxBytes, uint64_t maxFiles, std::set<std::string> retrieveRequestsToSkip);
+ CandidateJobList getCandidateList(uint64_t maxBytes, uint64_t maxFiles, const std::set<std::string> & retrieveRequestsToSkip,
+ const std::set<std::string> & diskSystemsToSkip);
+
//! Return a summary of the number of jobs and number of bytes in the queue
CandidateJobList getCandidateSummary();
@@ -117,6 +132,11 @@ public:
// -- Generic parameters
std::string getVid();
+ // Support for sleep waiting free space (back pressure).
+ // This data is queried through getJobsSummary().
+ void setSleepForFreeSpaceStartTimeAndName(time_t time, const std::string & diskSystemName, uint64_t sleepTime);
+ void resetSleepForFreeSpaceStartTime();
+
private:
struct ShardForAddition {
bool newShard=false;
@@ -157,12 +177,11 @@ private:
uint64_t m_maxShardSize = c_defaultMaxShardSize;
};
-class RetrieveQueueToTransferForUser : public RetrieveQueue { using RetrieveQueue::RetrieveQueue; };
+class RetrieveQueueToTransfer : public RetrieveQueue { using RetrieveQueue::RetrieveQueue; };
class RetrieveQueueToReportForUser : public RetrieveQueue { using RetrieveQueue::RetrieveQueue; };
class RetrieveQueueFailed : public RetrieveQueue { using RetrieveQueue::RetrieveQueue; };
class RetrieveQueueToReportToRepackForSuccess : public RetrieveQueue { using RetrieveQueue::RetrieveQueue; };
class RetrieveQueueToReportToRepackForFailure: public RetrieveQueue { using RetrieveQueue::RetrieveQueue; };
-class RetrieveQueueToTransferForRepack : public RetrieveQueue { using RetrieveQueue::RetrieveQueue; };
}}
diff --git a/objectstore/RetrieveQueueAlgorithms.hpp b/objectstore/RetrieveQueueAlgorithms.hpp
index 45dd12db5e3914b59ba2ae0c082e95f00d9530f0..2146f17783d857a81e989b6c2731bcafb973c8fe 100644
--- a/objectstore/RetrieveQueueAlgorithms.hpp
+++ b/objectstore/RetrieveQueueAlgorithms.hpp
@@ -31,7 +31,7 @@ struct ContainerTraits<RetrieveQueue,C>
ContainerSummary() : RetrieveQueue::JobsSummary() {}
ContainerSummary(const RetrieveQueue::JobsSummary &c) :
RetrieveQueue::JobsSummary({c.jobs,c.bytes,c.oldestJobStartTime,c.priority,
- c.minRetrieveRequestAge,c.maxDrivesAllowed,c.activityCounts}) {}
+ c.minRetrieveRequestAge,c.maxDrivesAllowed,c.activityCounts, nullopt}) {}
void addDeltaToLog(const ContainerSummary&, log::ScopedParamContainer&) const;
};
@@ -45,6 +45,7 @@ struct ContainerTraits<RetrieveQueue,C>
cta::common::dataStructures::MountPolicy policy;
serializers::RetrieveJobStatus status;
optional<RetrieveActivityDescription> activityDescription;
+ optional<std::string> diskSystemName;
typedef std::list<InsertedElement> list;
};
@@ -59,6 +60,8 @@ struct ContainerTraits<RetrieveQueue,C>
std::string errorReportURL;
SchedulerDatabase::RetrieveJob::ReportType reportType;
RetrieveRequest::RepackInfo repackInfo;
+ optional<RetrieveQueue::JobDump::ActivityDescription> activity;
+ optional<std::string> diskSystemName;
};
struct PoppedElementsSummary;
struct PopCriteria {
@@ -279,7 +282,7 @@ addReferencesAndCommit(Container &cont, typename InsertedElement::list &elemMemC
std::list<RetrieveQueue::JobToAdd> jobsToAdd;
for (auto &e : elemMemCont) {
RetrieveRequest &rr = *e.retrieveRequest;
- jobsToAdd.push_back({e.copyNb, e.fSeq, rr.getAddressIfSet(), e.filesize, e.policy, ::time(nullptr), e.activityDescription});
+ jobsToAdd.push_back({e.copyNb, e.fSeq, rr.getAddressIfSet(), e.filesize, e.policy, ::time(nullptr), e.activityDescription, e.diskSystemName});
}
cont.addJobsAndCommit(jobsToAdd, agentRef, lc);
}
@@ -292,7 +295,7 @@ addReferencesIfNecessaryAndCommit(Container& cont, typename InsertedElement::lis
std::list<RetrieveQueue::JobToAdd> jobsToAdd;
for (auto &e : elemMemCont) {
RetrieveRequest &rr = *e.retrieveRequest;
- jobsToAdd.push_back({e.copyNb, e.fSeq, rr.getAddressIfSet(), e.filesize, e.policy, ::time(nullptr), e.activityDescription});
+ jobsToAdd.push_back({e.copyNb, e.fSeq, rr.getAddressIfSet(), e.filesize, e.policy, ::time(nullptr), e.activityDescription, e.diskSystemName});
}
cont.addJobsIfNecessaryAndCommit(jobsToAdd, agentRef, lc);
}
@@ -387,6 +390,11 @@ switchElementsOwnership(PoppedElementsBatch &poppedElementBatch, const Container
e.archiveFile = u.get()->getArchiveFile();
e.rr = u.get()->getRetrieveRequest();
e.repackInfo = u.get()->getRepackInfo();
+ auto & rad = u.get()->getRetrieveActivityDescription();
+ if (rad) {
+ e.activity = RetrieveQueue::JobDump::ActivityDescription{ rad.value().diskInstanceName, rad.value().activity };
+ }
+ e.diskSystemName = u.get()->getDiskSystemName();
switch(u.get()->getJobStatus()) {
case serializers::RetrieveJobStatus::RJS_ToReportToUserForFailure:
e.reportType = SchedulerDatabase::RetrieveJob::ReportType::FailureReport;
@@ -408,7 +416,19 @@ bool ContainerTraits<RetrieveQueue,C>::
trimContainerIfNeeded(Container &cont, ScopedExclusiveLock &contLock,
const ContainerIdentifier &cId, log::LogContext &lc)
{
- if(!cont.isEmpty()) return false;
+ if(!cont.isEmpty()) {
+ auto si = cont.getJobsSummary().sleepInfo;
+ if (si) {
+ log::ScopedParamContainer params(lc);
+ params.add("tapeVid", cId)
+ .add("queueObject", cont.getAddressIfSet())
+ .add("diskSystemSleptFor", si.value().diskSystemSleptFor);
+ lc.log(log::INFO, "In ContainerTraits<RetrieveQueue,C>::trimContainerIfNeeded(): non-empty queue is sleeping");
+ // We fake the fact that we trimed the queue for compatibility with previous algorithms (a sleeping queue is like gone at this point).
+ return true;
+ }
+ return false;
+ }
// The current implementation is done unlocked
contLock.release();
try {
@@ -437,7 +457,7 @@ trimContainerIfNeeded(Container &cont, ScopedExclusiveLock &contLock,
// RetrieveQueue full specialisations for ContainerTraits.
template<>
-struct ContainerTraits<RetrieveQueue,RetrieveQueueToTransferForUser>::PopCriteria {
+struct ContainerTraits<RetrieveQueue,RetrieveQueueToTransfer>::PopCriteria {
uint64_t files;
uint64_t bytes;
PopCriteria(uint64_t f = 0, uint64_t b = 0) : files(f), bytes(b) {}
@@ -447,12 +467,20 @@ struct ContainerTraits<RetrieveQueue,RetrieveQueueToTransferForUser>::PopCriteri
files -= pes.files;
return *this;
}
+ struct DiskSystemToSkip {
+ std::string name;
+ uint64_t sleepTime;
+ bool operator<(const DiskSystemToSkip o) const { return name < o.name; }
+ };
+ std::set<DiskSystemToSkip> diskSystemsToSkip;
};
template<>
-struct ContainerTraits<RetrieveQueue,RetrieveQueueToTransferForUser>::PoppedElementsSummary {
+struct ContainerTraits<RetrieveQueue,RetrieveQueueToTransfer>::PoppedElementsSummary {
uint64_t files;
uint64_t bytes;
+ bool diskSystemFull = false;
+ std::string fullDiskSystem;
PoppedElementsSummary(uint64_t f = 0, uint64_t b = 0) : files(f), bytes(b) {}
bool operator==(const PoppedElementsSummary &pes) const {
return bytes == pes.bytes && files == pes.files;
@@ -478,12 +506,18 @@ struct ContainerTraits<RetrieveQueue,RetrieveQueueToTransferForUser>::PoppedElem
template<typename C>
auto ContainerTraits<RetrieveQueue,C>::
-getPoppingElementsCandidates(Container &cont, PopCriteria &unfulfilledCriteria, ElementsToSkipSet &elemtsToSkip,
+getPoppingElementsCandidates(Container &cont, PopCriteria &unfulfilledCriteria, ElementsToSkipSet &elementsToSkip,
log::LogContext &lc) -> PoppedElementsBatch
{
PoppedElementsBatch ret;
- auto candidateJobsFromQueue = cont.getCandidateList(std::numeric_limits<uint64_t>::max(), unfulfilledCriteria.files, elemtsToSkip);
+ auto candidateJobsFromQueue = cont.getCandidateList(std::numeric_limits<uint64_t>::max(), unfulfilledCriteria.files,
+ elementsToSkip,
+ // This parameter is needed only in the specialized version:
+ // auto ContainerTraits<RetrieveQueue,RetrieveQueueToTransfer>::getPoppingElementsCandidates
+ // We provide an empty set here.
+ std::set<std::string>()
+ );
for(auto &cjfq : candidateJobsFromQueue.candidates) {
ret.elements.emplace_back(PoppedElement{
cta::make_unique<RetrieveRequest>(cjfq.address, cont.m_objectStore),
@@ -492,7 +526,7 @@ getPoppingElementsCandidates(Container &cont, PopCriteria &unfulfilledCriteria,
common::dataStructures::ArchiveFile(),
common::dataStructures::RetrieveRequest(),
"", SchedulerDatabase::RetrieveJob::ReportType::NoReportRequired,
- RetrieveRequest::RepackInfo()
+ RetrieveRequest::RepackInfo(), cjfq.activity, cjfq.diskSystemName
});
ret.summary.files++;
}
@@ -503,7 +537,7 @@ template<typename C>
const std::string ContainerTraits<RetrieveQueue,C>::c_identifierType = "tapeVid";
template<>
-struct ContainerTraits<RetrieveQueue,RetrieveQueueToTransferForUser>::QueueType{
+struct ContainerTraits<RetrieveQueue,RetrieveQueueToTransfer>::QueueType{
objectstore::JobQueueType value = objectstore::JobQueueType::JobsToTransferForUser;
};
@@ -532,4 +566,9 @@ struct ContainerTraits<RetrieveQueue, RetrieveQueueToTransferForRepack>::QueueTy
objectstore::JobQueueType value = objectstore::JobQueueType::JobsToTransferForRepack;
};
+template<>
+auto ContainerTraits<RetrieveQueue,RetrieveQueueToTransfer>::
+getPoppingElementsCandidates(Container &cont, PopCriteria &unfulfilledCriteria, ElementsToSkipSet &elementsToSkip,
+ log::LogContext &lc) -> PoppedElementsBatch;
+
}} // namespace cta::objectstore
diff --git a/objectstore/RetrieveQueueShard.cpp b/objectstore/RetrieveQueueShard.cpp
index 76ebe1e5c36e03698b3c8f7894f8dbade013fe38..f28cbf33fccdf66663ff95a4d4b038a6429e1ef8 100644
--- a/objectstore/RetrieveQueueShard.cpp
+++ b/objectstore/RetrieveQueueShard.cpp
@@ -62,14 +62,23 @@ void RetrieveQueueShard::garbageCollect(const std::string& presumedOwner, AgentR
throw exception::Exception("In RetrieveQueueShard::garbageCollect(): garbage collection should not be necessary for this type of object.");
}
-RetrieveQueue::CandidateJobList RetrieveQueueShard::getCandidateJobList(uint64_t maxBytes, uint64_t maxFiles, std::set<std::string> retrieveRequestsToSkip) {
+RetrieveQueue::CandidateJobList RetrieveQueueShard::getCandidateJobList(uint64_t maxBytes, uint64_t maxFiles, const std::set<std::string> & retrieveRequestsToSkip, const std::set<std::string> & diskSystemsToSkip) {
checkPayloadReadable();
RetrieveQueue::CandidateJobList ret;
ret.remainingBytesAfterCandidates = m_payload.retrievejobstotalsize();
ret.remainingFilesAfterCandidates = m_payload.retrievejobs_size();
for (auto & j: m_payload.retrievejobs()) {
- if (!retrieveRequestsToSkip.count(j.address())) {
- ret.candidates.push_back({j.address(), (uint16_t)j.copynb(), j.size()});
+ if (!retrieveRequestsToSkip.count(j.address()) && !diskSystemsToSkip.count(j.destination_disk_system_name())) {
+ ret.candidates.push_back({j.address(), (uint16_t)j.copynb(), j.size(), nullopt, nullopt});
+ if (j.has_activity()) {
+ RetrieveQueue::JobDump::ActivityDescription ad;
+ ad.activity = j.activity();
+ ad.diskInstanceName = j.disk_instance_name();
+ ret.candidates.back().activity = ad;
+ }
+ if (j.has_destination_disk_system_name()) {
+ ret.candidates.back().diskSystemName = j.destination_disk_system_name();
+ }
ret.candidateBytes += j.size();
ret.candidateFiles ++;
}
@@ -104,7 +113,9 @@ auto RetrieveQueueShard::removeJobs(const std::list<std::string>& jobsToRemove)
ret.removedJobs.back().size = j.size();
ret.removedJobs.back().startTime = j.starttime();
if (j.has_activity())
- ret.removedJobs.back().activityDescription = JobInfo::ActivityDescription{ j.disk_instance_name(), j.activity() };
+ ret.removedJobs.back().activityDescription = RetrieveQueue::JobDump::ActivityDescription{ j.disk_instance_name(), j.activity() };
+ if (j.has_destination_disk_system_name())
+ ret.removedJobs.back().diskSystemName = j.destination_disk_system_name();
ret.bytesRemoved += j.size();
totalSize -= j.size();
ret.jobsRemoved++;
@@ -139,9 +150,12 @@ auto RetrieveQueueShard::dumpJobs() -> std::list<JobInfo> {
std::list<JobInfo> ret;
for (auto &j: m_payload.retrievejobs()) {
ret.emplace_back(JobInfo{j.size(), j.address(), (uint16_t)j.copynb(), j.priority(),
- j.minretrieverequestage(), j.maxdrivesallowed(), (time_t)j.starttime(), j.fseq(), nullopt});
+ j.minretrieverequestage(), j.maxdrivesallowed(), (time_t)j.starttime(), j.fseq(), nullopt, nullopt});
if (j.has_activity()) {
- ret.back().activityDescription = JobInfo::ActivityDescription{ j.disk_instance_name(), j.activity() };
+ ret.back().activityDescription = RetrieveQueue::JobDump::ActivityDescription{ j.disk_instance_name(), j.activity() };
+ }
+ if (j.has_destination_disk_system_name()) {
+ ret.back().diskSystemName = j.destination_disk_system_name();
}
}
return ret;
@@ -166,6 +180,8 @@ std::list<RetrieveQueue::JobToAdd> RetrieveQueueShard::dumpJobsToAdd() {
rad.activity = j.activity();
ret.back().activityDescription = rad;
}
+ if (j.has_destination_disk_system_name())
+ ret.back().diskSystemName = j.destination_disk_system_name();
}
return ret;
}
@@ -268,6 +284,7 @@ void RetrieveQueueShard::addJob(const RetrieveQueue::JobToAdd& jobToAdd) {
j->set_disk_instance_name(jobToAdd.activityDescription.value().diskInstanceName);
j->set_activity(jobToAdd.activityDescription.value().activity);
}
+ if (jobToAdd.diskSystemName) j->set_destination_disk_system_name(jobToAdd.diskSystemName.value());
m_payload.set_retrievejobstotalsize(m_payload.retrievejobstotalsize()+jobToAdd.fileSize);
// Sort the shard
size_t jobIndex = m_payload.retrievejobs_size() - 1;
@@ -304,6 +321,7 @@ void RetrieveQueueShard::addJobsThroughCopy(JobsToAddSet& jobsToAdd) {
rjp.set_disk_instance_name(jobToAdd.activityDescription.value().diskInstanceName);
rjp.set_activity(jobToAdd.activityDescription.value().activity);
}
+ if (jobToAdd.diskSystemName) rjp.set_destination_disk_system_name(jobToAdd.diskSystemName.value());
i = serializedJobsToAdd.insert(i, rjp);
totalSize+=jobToAdd.fileSize;
}
diff --git a/objectstore/RetrieveQueueShard.hpp b/objectstore/RetrieveQueueShard.hpp
index 7da9d10d72144578e2975e25332770e85d0b489b..fa798e4b439fca79104282dcb501b2ec0b53cdec 100644
--- a/objectstore/RetrieveQueueShard.hpp
+++ b/objectstore/RetrieveQueueShard.hpp
@@ -53,11 +53,8 @@ public:
uint64_t maxDrivesAllowed;
time_t startTime;
uint64_t fSeq;
- struct ActivityDescription {
- std::string diskInstanceName;
- std::string activity;
- };
- optional<ActivityDescription> activityDescription;
+ optional<RetrieveQueue::JobDump::ActivityDescription> activityDescription;
+ optional<std::string> diskSystemName;
};
std::list<JobInfo> dumpJobs();
@@ -115,7 +112,8 @@ public:
*/
RemovalResult removeJobs(const std::list<std::string> & jobsToRemove);
- RetrieveQueue::CandidateJobList getCandidateJobList(uint64_t maxBytes, uint64_t maxFiles, std::set<std::string> retrieveRequestsToSkip);
+ RetrieveQueue::CandidateJobList getCandidateJobList(uint64_t maxBytes, uint64_t maxFiles,
+ const std::set<std::string> & retrieveRequestsToSkip, const std::set<std::string> & diskSystemsToSkip);
/** Re compute summaries in case they do not match the array content. */
void rebuild();
diff --git a/objectstore/RetrieveQueueTest.cpp b/objectstore/RetrieveQueueTest.cpp
index 098114e6289ee943527374f1714ef091042cd2de..aa7d386797eae835bccd7a8954eaf32a7c1a7916 100644
--- a/objectstore/RetrieveQueueTest.cpp
+++ b/objectstore/RetrieveQueueTest.cpp
@@ -124,7 +124,8 @@ TEST(ObjectStore, RetrieveQueueShardingAndOrderingTest) {
ASSERT_EQ(minStartTime, rq.getJobsSummary().oldestJobStartTime);
uint64_t nextExpectedFseq=0;
while (rq.getJobsSummary().jobs) {
- auto candidateJobs = rq.getCandidateList(std::numeric_limits<uint64_t>::max(), 50, std::set<std::string>());
+ auto candidateJobs = rq.getCandidateList(std::numeric_limits<uint64_t>::max(), 50, std::set<std::string>(),
+ std::set<std::string>());
std::set<std::string> jobsToSkip;
std::list<std::string> jobsToDelete;
for (auto &j: candidateJobs.candidates) {
@@ -135,7 +136,7 @@ TEST(ObjectStore, RetrieveQueueShardingAndOrderingTest) {
jobsToDelete.emplace_back(j.address);
nextExpectedFseq++;
}
- auto candidateJobs2 = rq.getCandidateList(std::numeric_limits<uint64_t>::max(), 1, jobsToSkip);
+ auto candidateJobs2 = rq.getCandidateList(std::numeric_limits<uint64_t>::max(), 1, jobsToSkip, std::set<std::string>());
if (candidateJobs2.candidateFiles) {
std::stringstream address;
address << "someRequest-" << nextExpectedFseq;
@@ -245,7 +246,8 @@ TEST(ObjectStore, RetrieveQueueActivityCounts) {
ASSERT_EQ(0.2, jsB->weight);
uint64_t nextExpectedFseq=0;
while (rq.getJobsSummary().jobs) {
- auto candidateJobs = rq.getCandidateList(std::numeric_limits<uint64_t>::max(), 50, std::set<std::string>());
+ auto candidateJobs = rq.getCandidateList(std::numeric_limits<uint64_t>::max(), 50, std::set<std::string>(),
+ std::set<std::string>());
std::set<std::string> jobsToSkip;
std::list<std::string> jobsToDelete;
for (auto &j: candidateJobs.candidates) {
@@ -256,7 +258,7 @@ TEST(ObjectStore, RetrieveQueueActivityCounts) {
jobsToDelete.emplace_back(j.address);
nextExpectedFseq++;
}
- auto candidateJobs2 = rq.getCandidateList(std::numeric_limits<uint64_t>::max(), 1, jobsToSkip);
+ auto candidateJobs2 = rq.getCandidateList(std::numeric_limits<uint64_t>::max(), 1, jobsToSkip, std::set<std::string>());
if (candidateJobs2.candidateFiles) {
std::stringstream address;
address << "someRequest-" << nextExpectedFseq;
diff --git a/objectstore/RetrieveQueueToTransferAlgorithms.cpp b/objectstore/RetrieveQueueToTransferAlgorithms.cpp
index 2d80b46b33cf75d87886f1b2572b3f7739547096..679db13a4a74bea3fff25404c189e2a7b8a105f4 100644
--- a/objectstore/RetrieveQueueToTransferAlgorithms.cpp
+++ b/objectstore/RetrieveQueueToTransferAlgorithms.cpp
@@ -21,10 +21,10 @@
namespace cta { namespace objectstore {
template<>
-const std::string ContainerTraits<RetrieveQueue,RetrieveQueueToTransferForUser>::c_containerTypeName = "RetrieveQueueToTransferForUser";
+const std::string ContainerTraits<RetrieveQueue,RetrieveQueueToTransfer>::c_containerTypeName = "RetrieveQueueToTransfer";
template<>
-auto ContainerTraits<RetrieveQueue,RetrieveQueueToTransferForUser>::
+auto ContainerTraits<RetrieveQueue,RetrieveQueueToTransfer>::
getElementSummary(const PoppedElement &poppedElement) -> PoppedElementsSummary {
PoppedElementsSummary ret;
ret.bytes = poppedElement.bytes;
@@ -33,20 +33,36 @@ getElementSummary(const PoppedElement &poppedElement) -> PoppedElementsSummary {
}
template<>
-void ContainerTraits<RetrieveQueue,RetrieveQueueToTransferForUser>::PoppedElementsBatch::
+void ContainerTraits<RetrieveQueue,RetrieveQueueToTransfer>::PoppedElementsBatch::
addToLog(log::ScopedParamContainer ¶ms) const {
params.add("bytes", summary.bytes)
.add("files", summary.files);
}
template<>
-auto ContainerTraits<RetrieveQueue,RetrieveQueueToTransferForUser>::
-getPoppingElementsCandidates(Container &cont, PopCriteria &unfulfilledCriteria, ElementsToSkipSet &elemtsToSkip,
+auto ContainerTraits<RetrieveQueue,RetrieveQueueToTransfer>::
+getPoppingElementsCandidates(Container &cont, PopCriteria &unfulfilledCriteria, ElementsToSkipSet &elementsToSkip,
log::LogContext &lc) -> PoppedElementsBatch
{
PoppedElementsBatch ret;
- auto candidateJobsFromQueue = cont.getCandidateList(unfulfilledCriteria.bytes, unfulfilledCriteria.files, elemtsToSkip);
+ std::set<std::string> diskSystemsToSkipNames;
+ for (auto &ds: unfulfilledCriteria.diskSystemsToSkip) diskSystemsToSkipNames.insert(ds.name);
+ auto candidateJobsFromQueue = cont.getCandidateList(unfulfilledCriteria.bytes, unfulfilledCriteria.files, elementsToSkip, diskSystemsToSkipNames);
+ if (unfulfilledCriteria.diskSystemsToSkip.size() && candidateJobsFromQueue.candidates.empty() && cont.getJobsSummary().jobs && elementsToSkip.empty()) {
+ // We failed to find any candidates from a non empty queue, from which there are no individual elements to skip.
+ // this means the it is time to sleep this queue.
+ // We log and return empty. Caller will stop trying to pop after that.
+ cont.setSleepForFreeSpaceStartTimeAndName(::time(nullptr), unfulfilledCriteria.diskSystemsToSkip.begin()->name, unfulfilledCriteria.diskSystemsToSkip.begin()->sleepTime);
+ cont.commit();
+ log::ScopedParamContainer params(lc);
+ params.add("tapeVid", cont.getVid())
+ .add("queueObject", cont.getAddressIfSet())
+ .add("diskSystemName", unfulfilledCriteria.diskSystemsToSkip.begin()->name)
+ .add("sleepTime", unfulfilledCriteria.diskSystemsToSkip.begin()->sleepTime);
+ lc.log(log::WARNING, "In ContainerTraits<RetrieveQueue,RetrieveQueueToTransfer>::getPoppingElementsCandidates(): sleeping queue due to disk system full.");
+ return ret;
+ }
for(auto &cjfq : candidateJobsFromQueue.candidates) {
ret.elements.emplace_back(PoppedElement{
cta::make_unique<RetrieveRequest>(cjfq.address, cont.m_objectStore),
@@ -56,7 +72,9 @@ getPoppingElementsCandidates(Container &cont, PopCriteria &unfulfilledCriteria,
common::dataStructures::RetrieveRequest(),
"",
SchedulerDatabase::RetrieveJob::ReportType::NoReportRequired,
- RetrieveRequest::RepackInfo()
+ RetrieveRequest::RepackInfo(),
+ cjfq.activity,
+ cjfq.diskSystemName
});
ret.summary.bytes += cjfq.size;
ret.summary.files++;
diff --git a/objectstore/RetrieveQueueToTransferForRepackAlgorithms.cpp b/objectstore/RetrieveQueueToTransferForRepackAlgorithms.cpp
deleted file mode 100644
index 18e8491cc24219ed0acbfea4013ea047a0713ffd..0000000000000000000000000000000000000000
--- a/objectstore/RetrieveQueueToTransferForRepackAlgorithms.cpp
+++ /dev/null
@@ -1,25 +0,0 @@
-/**
- * The CERN Tape Archive (CTA) project
- * Copyright © 2018 CERN
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-#include "RetrieveQueueAlgorithms.hpp"
-
-namespace cta { namespace objectstore {
-
-template<>
-const std::string ContainerTraits<RetrieveQueue,RetrieveQueueToTransferForRepack>::c_containerTypeName = "RetrieveQueueToTransferForRepack";
-
-}}
\ No newline at end of file
diff --git a/objectstore/RetrieveRequest.cpp b/objectstore/RetrieveRequest.cpp
index 672c5f5bb32066488f950ffd8ab27db34c6f33ee..2a77db5af1630508a78f6d9848e8c1d48389d107 100644
--- a/objectstore/RetrieveRequest.cpp
+++ b/objectstore/RetrieveRequest.cpp
@@ -87,7 +87,7 @@ void RetrieveRequest::garbageCollect(const std::string& presumedOwner, AgentRefe
std::set<std::string> candidateVids;
for (auto &j: m_payload.jobs()) {
switch(j.status()){
- case RetrieveJobStatus::RJS_ToTransferForUser:
+ case RetrieveJobStatus::RJS_ToTransfer:
// Find the job details in tape file
for (auto &tf: m_payload.archivefile().tapefiles()) {
if (tf.copynb() == j.copynb()) {
@@ -160,7 +160,7 @@ queueForFailure:;
{
// If there is no candidate, we fail the jobs that are not yet, and queue the request as failed (on any VID).
for (auto & j: *m_payload.mutable_jobs()) {
- if (j.status() == RetrieveJobStatus::RJS_ToTransferForUser) {
+ if (j.status() == RetrieveJobStatus::RJS_ToTransfer) {
j.set_status(RetrieveJobStatus::RJS_Failed);
log::ScopedParamContainer params(lc);
params.add("fileId", m_payload.archivefile().archivefileid())
@@ -204,7 +204,7 @@ queueForFailure:;
objectstore::MountPolicySerDeser mp;
std::list<RetrieveQueue::JobToAdd> jta;
jta.push_back({activeCopyNb, activeFseq, getAddressIfSet(), m_payload.archivefile().filesize(),
- mp, (signed)m_payload.schedulerrequest().entrylog().time(), nullopt});
+ mp, (signed)m_payload.schedulerrequest().entrylog().time(), nullopt, nullopt});
if (m_payload.has_activity_weight()) {
RetrieveActivityDescription activityDescription;
activityDescription.priority = m_payload.activity_weight().priority();
@@ -272,7 +272,7 @@ queueForTransfer:;
mp.deserialize(m_payload.mountpolicy());
std::list<RetrieveQueue::JobToAdd> jta;
jta.push_back({bestTapeFile->copynb(), bestTapeFile->fseq(), getAddressIfSet(), m_payload.archivefile().filesize(),
- mp, (signed)m_payload.schedulerrequest().entrylog().time(), nullopt});
+ mp, (signed)m_payload.schedulerrequest().entrylog().time(), getActivity(), getDiskSystemName()});
if (m_payload.has_activity_weight()) {
RetrieveActivityDescription activityDescription;
activityDescription.priority = m_payload.activity_weight().priority();
@@ -348,7 +348,7 @@ void RetrieveRequest::addJob(uint32_t copyNb, uint16_t maxRetriesWithinMount, ui
tf->set_totalretries(0);
tf->set_maxreportretries(maxReportRetries);
tf->set_totalreportretries(0);
- tf->set_status(serializers::RetrieveJobStatus::RJS_ToTransferForUser);
+ tf->set_status(serializers::RetrieveJobStatus::RJS_ToTransfer);
}
//------------------------------------------------------------------------------
@@ -375,7 +375,7 @@ auto RetrieveRequest::addTransferFailure(uint32_t copyNumber, uint64_t mountId,
}
if(j.totalretries() < j.maxtotalretries()) {
EnqueueingNextStep ret;
- ret.nextStatus = serializers::RetrieveJobStatus::RJS_ToTransferForUser;
+ ret.nextStatus = serializers::RetrieveJobStatus::RJS_ToTransfer;
if(j.retrieswithinmount() < j.maxretrieswithinmount())
// Job can try again within this mount
ret.nextStep = EnqueueingNextStep::NextStep::EnqueueForTransferForUser;
@@ -538,6 +538,25 @@ optional<RetrieveActivityDescription> RetrieveRequest::getActivity() {
return ret;
}
+//------------------------------------------------------------------------------
+// RetrieveRequest::setDiskSystemName()
+//------------------------------------------------------------------------------
+void RetrieveRequest::setDiskSystemName(const std::string& diskSystemName) {
+ checkPayloadWritable();
+ m_payload.set_disk_system_name(diskSystemName);
+}
+
+//------------------------------------------------------------------------------
+// RetrieveRequest::getDiskSystemName()
+//------------------------------------------------------------------------------
+optional<std::string> RetrieveRequest::getDiskSystemName() {
+ checkPayloadReadable();
+ optional<std::string> ret;
+ if (m_payload.has_disk_system_name())
+ ret = m_payload.disk_system_name();
+ return ret;
+}
+
//------------------------------------------------------------------------------
// RetrieveRequest::dumpJobs()
//------------------------------------------------------------------------------
@@ -608,10 +627,10 @@ bool RetrieveRequest::addJobFailure(uint32_t copyNumber, uint64_t mountId,
if (j.totalretries() >= j.maxtotalretries()) {
j.set_status(serializers::RetrieveJobStatus::RJS_ToReportToUserForFailure);
for (auto & j2: m_payload.jobs())
- if (j2.status() == serializers::RetrieveJobStatus::RJS_ToTransferForUser) return false;
+ if (j2.status() == serializers::RetrieveJobStatus::RJS_ToTransfer) return false;
return true;
} else {
- j.set_status(serializers::RetrieveJobStatus::RJS_ToTransferForUser);
+ j.set_status(serializers::RetrieveJobStatus::RJS_ToTransfer);
return false;
}
}
@@ -680,7 +699,7 @@ JobQueueType RetrieveRequest::getQueueType() {
for (auto &j: m_payload.jobs()) {
// Any job is to be transfered => To transfer
switch(j.status()) {
- case serializers::RetrieveJobStatus::RJS_ToTransferForUser:
+ case serializers::RetrieveJobStatus::RJS_ToTransfer:
return JobQueueType::JobsToTransferForUser;
break;
case serializers::RetrieveJobStatus::RJS_ToReportToRepackForSuccess:
@@ -692,8 +711,6 @@ JobQueueType RetrieveRequest::getQueueType() {
break;
case serializers::RetrieveJobStatus::RJS_ToReportToRepackForFailure:
return JobQueueType::JobsToReportToRepackForFailure;
- case serializers::RetrieveJobStatus::RJS_ToTransferForRepack:
- return JobQueueType::JobsToTransferForRepack;
default: break;
}
}
@@ -701,12 +718,15 @@ JobQueueType RetrieveRequest::getQueueType() {
return JobQueueType::FailedJobs;
}
+//------------------------------------------------------------------------------
+// RetrieveRequest::getQueueType()
+//------------------------------------------------------------------------------
JobQueueType RetrieveRequest::getQueueType(uint32_t copyNb){
checkPayloadReadable();
for(auto &j: m_payload.jobs()){
if(j.copynb() == copyNb){
switch(j.status()){
- case serializers::RetrieveJobStatus::RJS_ToTransferForUser:
+ case serializers::RetrieveJobStatus::RJS_ToTransfer:
return JobQueueType::JobsToTransferForUser;
case serializers::RetrieveJobStatus::RJS_ToReportToRepackForSuccess:
return JobQueueType::JobsToReportToRepackForSuccess;
@@ -716,8 +736,6 @@ JobQueueType RetrieveRequest::getQueueType(uint32_t copyNb){
return JobQueueType::FailedJobs;
case serializers::RetrieveJobStatus::RJS_ToReportToRepackForFailure:
return JobQueueType::JobsToReportToRepackForFailure;
- case serializers::RetrieveJobStatus::RJS_ToTransferForRepack:
- return JobQueueType::JobsToTransferForRepack;
default:
return JobQueueType::FailedJobs;
}
@@ -731,7 +749,7 @@ JobQueueType RetrieveRequest::getQueueType(uint32_t copyNb){
//------------------------------------------------------------------------------
std::string RetrieveRequest::statusToString(const serializers::RetrieveJobStatus& status) {
switch(status) {
- case serializers::RetrieveJobStatus::RJS_ToTransferForUser:
+ case serializers::RetrieveJobStatus::RJS_ToTransfer:
return "ToTransfer";
case serializers::RetrieveJobStatus::RJS_Failed:
return "Failed";
@@ -779,7 +797,7 @@ auto RetrieveRequest::determineNextStep(uint32_t copyNumberUpdated, JobEvent job
switch (jobEvent)
{
case JobEvent::TransferFailed:
- if (*currentStatus != RetrieveJobStatus::RJS_ToTransferForUser) {
+ if (*currentStatus != RetrieveJobStatus::RJS_ToTransfer) {
// Wrong status, but the context leaves no ambiguity. Just warn.
log::ScopedParamContainer params(lc);
params.add("event", eventToString(jobEvent))
@@ -837,7 +855,7 @@ void RetrieveRequest::updateLifecycleTiming(serializers::RetrieveRequest& payloa
LifecycleTimingsSerDeser lifeCycleSerDeser;
lifeCycleSerDeser.deserialize(payload.lifecycle_timings());
switch(retrieveJob.status()){
- case RetrieveJobStatus::RJS_ToTransferForUser:
+ case RetrieveJobStatus::RJS_ToTransfer:
if(retrieveJob.totalretries() == 0){
//totalretries = 0 then this is the first selection of the request
lifeCycleSerDeser.first_selected_time = time(nullptr);
@@ -907,6 +925,15 @@ auto RetrieveRequest::asyncUpdateJobOwner(uint32_t copyNumber, const std::string
af.deserialize(payload.archivefile());
retRef.m_archiveFile = af;
retRef.m_jobStatus = j.status();
+ if (payload.has_activity_weight()) {
+ retRef.m_retrieveActivityDescription = RetrieveActivityDescription{
+ payload.activity_weight().priority(), payload.activity_weight().disk_instance_name(),
+ payload.activity_weight().activity(), payload.activity_weight().creation_time(),
+ payload.activity_weight().weight(), 0
+ };
+ }
+ if (payload.has_disk_system_name())
+ retRef.m_diskSystemName = payload.disk_system_name();
RetrieveRequest::updateLifecycleTiming(payload,j);
LifecycleTimingsSerDeser lifeCycleSerDeser;
lifeCycleSerDeser.deserialize(payload.lifecycle_timings());
@@ -958,6 +985,20 @@ const RetrieveRequest::RepackInfo& RetrieveRequest::AsyncJobOwnerUpdater::getRep
return m_repackInfo;
}
+//------------------------------------------------------------------------------
+// RetrieveRequest::AsyncJobOwnerUpdater::getRetrieveActivityDescription()
+//------------------------------------------------------------------------------
+const optional<RetrieveActivityDescription>& RetrieveRequest::AsyncJobOwnerUpdater::getRetrieveActivityDescription() {
+ return m_retrieveActivityDescription;
+}
+
+//------------------------------------------------------------------------------
+// RetrieveRequest::AsyncJobOwnerUpdater::getDiskSystemName()
+//------------------------------------------------------------------------------
+const optional<std::string>& RetrieveRequest::AsyncJobOwnerUpdater::getDiskSystemName() {
+ return m_diskSystemName;
+}
+
//------------------------------------------------------------------------------
// RetrieveRequest::AsyncJobOwnerUpdater::getRetrieveRequest()
//------------------------------------------------------------------------------
diff --git a/objectstore/RetrieveRequest.hpp b/objectstore/RetrieveRequest.hpp
index b93d054da669ef4c4d0db9d7309e262acde4fa7c..63c37f74a2efd7b3f7b8d76518723668bba00620 100644
--- a/objectstore/RetrieveRequest.hpp
+++ b/objectstore/RetrieveRequest.hpp
@@ -225,6 +225,8 @@ public:
const common::dataStructures::RetrieveRequest &getRetrieveRequest();
const common::dataStructures::ArchiveFile &getArchiveFile();
const RepackInfo &getRepackInfo();
+ const optional<RetrieveActivityDescription> &getRetrieveActivityDescription();
+ const optional<std::string> &getDiskSystemName();
private:
std::function<std::string(const std::string &)> m_updaterCallback;
std::unique_ptr<Backend::AsyncUpdater> m_backendUpdater;
@@ -232,6 +234,8 @@ public:
common::dataStructures::ArchiveFile m_archiveFile;
RepackInfo m_repackInfo;
serializers::RetrieveJobStatus m_jobStatus;
+ optional<RetrieveActivityDescription> m_retrieveActivityDescription;
+ optional<std::string> m_diskSystemName;
};
// An owner updater factory. The owner MUST be previousOwner for the update to be executed.
AsyncJobOwnerUpdater *asyncUpdateJobOwner(uint32_t copyNumber, const std::string &owner, const std::string &previousOwner);
@@ -243,6 +247,8 @@ public:
void setActivityIfNeeded(const cta::common::dataStructures::RetrieveRequest & retrieveRequest,
const cta::common::dataStructures::RetrieveFileQueueCriteria& criteria);
optional<RetrieveActivityDescription> getActivity();
+ void setDiskSystemName(const std::string & diskSystemName);
+ optional<std::string> getDiskSystemName();
cta::common::dataStructures::RetrieveFileQueueCriteria getRetrieveFileQueueCriteria();
cta::common::dataStructures::ArchiveFile getArchiveFile();
cta::common::dataStructures::EntryLog getEntryLog();
diff --git a/objectstore/Sorter.cpp b/objectstore/Sorter.cpp
index 4ea3365f41e3208b4a623f1334b98be22299a986..54b0809cca7cadb23be59a19fb58a675ccd16b3d 100644
--- a/objectstore/Sorter.cpp
+++ b/objectstore/Sorter.cpp
@@ -159,7 +159,7 @@ void Sorter::executeRetrieveAlgorithm(const std::string vid, std::string& queueA
Sorter::RetrieveJob job = std::get<0>(jobToAdd->jobToQueue);
succeededJobs[job.jobDump.copyNb] = jobToAdd;
previousOwner = job.previousOwner->getAgentAddress();
- jobsToAdd.push_back({job.retrieveRequest.get(),job.jobDump.copyNb,job.fSeq,job.fileSize,job.mountPolicy,job.jobDump.status,job.activityDescription});
+ jobsToAdd.push_back({job.retrieveRequest.get(),job.jobDump.copyNb,job.fSeq,job.fileSize,job.mountPolicy,job.jobDump.status,job.activityDescription,job.diskSystemName});
}
try{
algo.referenceAndSwitchOwnershipIfNecessary(vid,previousOwner,queueAddress,jobsToAdd,lc);
@@ -185,7 +185,7 @@ void Sorter::dispatchRetrieveAlgorithm(const std::string vid, const JobQueueType
this->executeRetrieveAlgorithm<RetrieveQueueToReportForUser>(vid,queueAddress,jobs,lc);
break;
case JobQueueType::JobsToTransferForUser:
- this->executeRetrieveAlgorithm<RetrieveQueueToTransferForUser>(vid,queueAddress,jobs,lc);
+ this->executeRetrieveAlgorithm<RetrieveQueueToTransfer>(vid,queueAddress,jobs,lc);
break;
case JobQueueType::JobsToReportToRepackForSuccess:
this->executeRetrieveAlgorithm<RetrieveQueueToReportToRepackForSuccess>(vid,queueAddress,jobs,lc);
@@ -193,8 +193,6 @@ void Sorter::dispatchRetrieveAlgorithm(const std::string vid, const JobQueueType
case JobQueueType::JobsToReportToRepackForFailure:
this->executeRetrieveAlgorithm<RetrieveQueueToReportToRepackForFailure>(vid,queueAddress,jobs,lc);
break;
- case JobQueueType::JobsToTransferForRepack:
- this->executeRetrieveAlgorithm<RetrieveQueueToTransferForRepack>(vid, queueAddress,jobs,lc);
break;
case JobQueueType::FailedJobs:
break;
@@ -216,6 +214,8 @@ Sorter::RetrieveJob Sorter::createRetrieveJob(std::shared_ptr<RetrieveRequest> r
jobToAdd.jobDump.status = retrieveRequest->getJobStatus(jobToAdd.jobDump.copyNb);
jobToAdd.fileSize = archiveFile.fileSize;
jobToAdd.jobQueueType = retrieveRequest->getQueueType(copyNb); //May throw an exception
+ jobToAdd.activityDescription = retrieveRequest->getActivity();
+ jobToAdd.diskSystemName = retrieveRequest->getDiskSystemName();
return jobToAdd;
}
@@ -308,7 +308,7 @@ std::set<std::string> Sorter::getCandidateVidsToTransfer(RetrieveRequestInfosAcc
using serializers::RetrieveJobStatus;
std::set<std::string> candidateVids;
for(auto& j: requestAccessor.getJobs()){
- if(j.status == RetrieveJobStatus::RJS_ToTransferForUser){
+ if(j.status == RetrieveJobStatus::RJS_ToTransfer){
candidateVids.insert(requestAccessor.getArchiveFile().tapeFiles.at(j.copyNb).vid);
}
}
@@ -401,6 +401,8 @@ Sorter::RetrieveJob OStoreRetrieveRequestAccessor::createRetrieveJob(const cta::
ret.jobDump.status = m_retrieveRequest->getJobStatus(ret.jobDump.copyNb);
ret.jobQueueType = m_retrieveRequest->getQueueType(copyNb);
ret.fileSize = archiveFile.fileSize;
+ ret.activityDescription = m_retrieveRequest->getActivity();
+ ret.diskSystemName = m_retrieveRequest->getDiskSystemName();
return ret;
}
diff --git a/objectstore/Sorter.hpp b/objectstore/Sorter.hpp
index 81dafeb0aaea21dc35be0f98ee0907fd1f764a53..d13967be86d0e7716257d71c1c6bcae4ea50dd91 100644
--- a/objectstore/Sorter.hpp
+++ b/objectstore/Sorter.hpp
@@ -107,6 +107,7 @@ public:
common::dataStructures::MountPolicy mountPolicy;
cta::objectstore::JobQueueType jobQueueType;
optional<RetrieveActivityDescription> activityDescription;
+ optional<std::string> diskSystemName;
};
/**
diff --git a/objectstore/SorterTest.cpp b/objectstore/SorterTest.cpp
index 5374d6139e3dc779ea7717735b64e274f0b6ee8f..13f63a0e61f544fd59b40d3dd6ac0bbf94805e43 100644
--- a/objectstore/SorterTest.cpp
+++ b/objectstore/SorterTest.cpp
@@ -264,7 +264,8 @@ TEST(ObjectStore,SorterInsertRetrieveRequest){
rqc.mountPolicy.retrievePriority = 1;
rr.setRetrieveFileQueueCriteria(rqc);
- rr.setJobStatus(2,cta::objectstore::serializers::RetrieveJobStatus::RJS_ToTransferForRepack);
+ // Make sure job 1 will get queued by failing the other one.
+ rr.setJobStatus(2,cta::objectstore::serializers::RetrieveJobStatus::RJS_Failed);
cta::common::dataStructures::RetrieveRequest sReq;
sReq.archiveFileID = rqc.archiveFile.archiveFileID;
@@ -298,7 +299,7 @@ TEST(ObjectStore,SorterInsertRetrieveRequest){
allFutures.clear();
- typedef ContainerAlgorithms<RetrieveQueue,RetrieveQueueToTransferForUser> Algo;
+ typedef ContainerAlgorithms<RetrieveQueue,RetrieveQueueToTransfer> Algo;
Algo algo(be,agentRef);
typename Algo::PopCriteria criteria;
@@ -326,6 +327,10 @@ TEST(ObjectStore,SorterInsertRetrieveRequest){
{
ScopedExclusiveLock sel(*retrieveRequest);
retrieveRequest->fetch();
+ // Make sure now copy 2 will get queued.
+ retrieveRequest->setJobStatus(1,cta::objectstore::serializers::RetrieveJobStatus::RJS_Failed);
+ retrieveRequest->setJobStatus(2,cta::objectstore::serializers::RetrieveJobStatus::RJS_ToTransfer);
+ retrieveRequest->commit();
ASSERT_NO_THROW(sorter.insertRetrieveRequest(retrieveRequest,agentRef,cta::optional<uint32_t>(2),lc));
@@ -345,11 +350,12 @@ TEST(ObjectStore,SorterInsertRetrieveRequest){
}
ASSERT_EQ(sorter.getAllRetrieve().size(),0);
- typedef ContainerAlgorithms<RetrieveQueue,RetrieveQueueToTransferForRepack> Algo;
+ typedef ContainerAlgorithms<RetrieveQueue,RetrieveQueueToTransfer> Algo;
Algo algo(be,agentRef);
typename Algo::PopCriteria criteria;
criteria.files = 1;
+ criteria.bytes = 1000;
typename Algo::PoppedElementsBatch elements = algo.popNextBatch("Tape1",criteria,lc);
ASSERT_EQ(elements.elements.size(),1);
auto& elt = elements.elements.front();
@@ -371,10 +377,14 @@ TEST(ObjectStore,SorterInsertRetrieveRequest){
{
ScopedExclusiveLock sel(*retrieveRequest);
retrieveRequest->fetch();
+ // We should be forbidden to force queueing a non-exsiting copy number.
ASSERT_THROW(sorter.insertRetrieveRequest(retrieveRequest,agentRef,cta::optional<uint32_t>(4),lc),cta::exception::Exception);
retrieveRequest->setJobStatus(1,serializers::RetrieveJobStatus::RJS_ToReportToRepackForSuccess);
+ retrieveRequest->setJobStatus(2,serializers::RetrieveJobStatus::RJS_Failed);
+ retrieveRequest->commit();
+ // We should be forbidden to requeue a request if no copy is in status ToTranfer.
ASSERT_THROW(sorter.insertRetrieveRequest(retrieveRequest,agentRef,cta::nullopt,lc),cta::exception::Exception);
sel.release();
@@ -630,7 +640,7 @@ TEST(ObjectStore,SorterInsertDifferentTypesOfRequests){
{
//Test the Retrieve Jobs
- typedef ContainerAlgorithms<RetrieveQueue,RetrieveQueueToTransferForUser> Algo;
+ typedef ContainerAlgorithms<RetrieveQueue,RetrieveQueueToTransfer> Algo;
Algo algo(be,agentRef);
typename Algo::PopCriteria criteria;
criteria.files = 2;
@@ -919,7 +929,7 @@ TEST(ObjectStore,SorterInsertRetrieveRequestNotFetched){
job.fSeq = tf.fSeq;
job.fileSize = rqc.archiveFile.fileSize;
job.jobDump.copyNb = tf.copyNb;
- job.jobDump.status = serializers::RetrieveJobStatus::RJS_ToTransferForUser;
+ job.jobDump.status = serializers::RetrieveJobStatus::RJS_ToTransfer;
job.jobQueueType = JobQueueType::JobsToTransferForUser;
job.mountPolicy = rqc.mountPolicy;
job.previousOwner = &agentRef;
@@ -946,7 +956,7 @@ TEST(ObjectStore,SorterInsertRetrieveRequestNotFetched){
{
//Test the Retrieve Jobs
- typedef ContainerAlgorithms<RetrieveQueue,RetrieveQueueToTransferForUser> Algo;
+ typedef ContainerAlgorithms<RetrieveQueue,RetrieveQueueToTransfer> Algo;
Algo algo(be,agentRef);
typename Algo::PopCriteria criteria;
criteria.files = 2;
diff --git a/objectstore/cta.proto b/objectstore/cta.proto
index 1a4272fb7cffc2fb08adef68467cf6c584f9848e..9523face634a1920c4781bc165d36f688880a188 100644
--- a/objectstore/cta.proto
+++ b/objectstore/cta.proto
@@ -156,6 +156,7 @@ message Agent {
required uint64 timeout_us = 2002;
repeated string ownedobjects = 2003;
optional bool being_garbage_collected = 2004 [default = false];
+ optional bool gc_needed = 2005 [default = false];
}
message AgentRegister {
@@ -210,6 +211,12 @@ message ArchiveFile {
// ------------- Drives handling ----------------------------------------------
+message DiskSpaceReservation {
+ // Each drive keeps tabs of its intended
+ required string disk_system_name = 5100;
+ required uint64 reserved_bytes = 5110;
+}
+
message DriveConfig {
required string category = 13000;
required string key = 13001;
@@ -242,7 +249,7 @@ message DriveState {
required bool desiredUp = 5019;
required bool desiredForceDown = 5020;
optional string currentvid = 5021;
- optional string cta_version = 5034;
+ optional string cta_version = 5035;
optional uint64 current_priority = 5028;
optional string current_activity = 5029;
optional double current_activity_weight = 5030;
@@ -253,9 +260,10 @@ message DriveState {
optional uint64 next_priority = 5031;
optional string next_activity = 5032;
optional double next_activity_weight = 5033;
- optional string dev_file_name = 5035;
- optional string raw_library_slot = 5036;
- repeated DriveConfig drive_config = 5037;
+ repeated DiskSpaceReservation disk_space_reservations = 5034;
+ optional string dev_file_name = 5036;
+ optional string raw_library_slot = 5037;
+ repeated DriveConfig drive_config = 5038;
// TODO: implement or remove required EntryLog creationlog = 5023;
}
@@ -367,12 +375,11 @@ message ArchiveRequest {
// of the request.
enum RetrieveJobStatus {
- RJS_ToTransferForUser = 1;
+ RJS_ToTransfer = 1;
RJS_ToReportToUserForFailure = 997;
RJS_Failed = 998;
RJS_ToReportToRepackForSuccess = 1002; //For Retrieve request created by a Repack request
RJS_ToReportToRepackForFailure = 1003;
- RJS_ToTransferForRepack = 1004;
}
message SchedulerRetrieveRequest {
@@ -439,6 +446,7 @@ message RetrieveRequest {
required bool isrepack = 9157;
optional RetrieveRequestRepackInfo repack_info = 9158;
optional LifecycleTimings lifecycle_timings = 9159;
+ optional string disk_system_name = 9161;
}
message ValueCountPair {
@@ -493,6 +501,7 @@ message RetrieveJobPointer {
// For activity (if present), we need disk instance and activity name (priority is always provided)
optional string disk_instance_name = 3109;
optional string activity = 3110;
+ optional string destination_disk_system_name = 3111;
}
message RetrieveQueueShardPointer {
@@ -525,6 +534,9 @@ message RetrieveQueue {
required uint64 oldestjobcreationtime = 10150;
required uint64 mapsrebuildcount = 10160;
required uint64 maxshardsize = 10170;
+ optional uint64 sleep_for_free_space_since = 10180;
+ optional string disk_system_slept_for = 10190;
+ optional uint64 sleep_time = 10200;
}
// ------------- Repack data strcutures ----------------------------------------
diff --git a/scheduler/OStoreDB/MemQueues.cpp b/scheduler/OStoreDB/MemQueues.cpp
index 7a1cafd3a8660db7dbc335a8894c1280838eb7b6..67c3172e1ae5d59710eb6687705b4483047cb17b 100644
--- a/scheduler/OStoreDB/MemQueues.cpp
+++ b/scheduler/OStoreDB/MemQueues.cpp
@@ -55,7 +55,7 @@ void MemQueue<objectstore::RetrieveRequest, objectstore::RetrieveQueue>::special
if (j.copyNb == job.copyNb) {
auto criteria = request.getRetrieveFileQueueCriteria();
jtal.push_back({j.copyNb, j.fSeq, request.getAddressIfSet(), criteria.archiveFile.fileSize,
- criteria.mountPolicy, request.getEntryLog().time, request.getActivity()});
+ criteria.mountPolicy, request.getEntryLog().time, request.getActivity(), request.getDiskSystemName()});
request.setActiveCopyNumber(j.copyNb);
request.setOwner(queueAddress);
goto jobAdded;
diff --git a/scheduler/OStoreDB/OStoreDB.cpp b/scheduler/OStoreDB/OStoreDB.cpp
index 5500f1eee6d350a1a57c56a8ba77356fd33cd75e..3aebfc90220b445f0735ad62567b02b6a375cac8 100644
--- a/scheduler/OStoreDB/OStoreDB.cpp
+++ b/scheduler/OStoreDB/OStoreDB.cpp
@@ -314,12 +314,12 @@ void OStoreDB::fetchMountInfo(SchedulerDatabase::TapeMountDecisionInfo& tmdi, Ro
auto & m = tmdi.potentialMounts.back();
m.vid = rqp.vid;
m.type = cta::common::dataStructures::MountType::Retrieve;
- m.bytesQueued = rqueue.getJobsSummary().bytes;
- m.filesQueued = rqueue.getJobsSummary().jobs;
+ m.bytesQueued = rqSummary.bytes;
+ m.filesQueued = rqSummary.jobs;
m.oldestJobStartTime = rqueue.getJobsSummary().oldestJobStartTime;
- m.priority = rqueue.getJobsSummary().priority;
- m.maxDrivesAllowed = rqueue.getJobsSummary().maxDrivesAllowed;
- m.minRequestAge = rqueue.getJobsSummary().minRetrieveRequestAge;
+ m.priority = rqSummary.priority;
+ m.maxDrivesAllowed = rqSummary.maxDrivesAllowed;
+ m.minRequestAge = rqSummary.minRetrieveRequestAge;
m.logicalLibrary = ""; // The logical library is not known here, and will be determined by the caller.
m.tapePool = ""; // The tape pool is not know and will be determined by the caller.
m.vendor = ""; // The vendor is not known here, and will be determined by the caller.
@@ -331,6 +331,15 @@ void OStoreDB::fetchMountInfo(SchedulerDatabase::TapeMountDecisionInfo& tmdi, Ro
m.activityNameAndWeightedMountCount.value().weight = ac.weight;
m.activityNameAndWeightedMountCount.value().weightedMountCount = 0.0; // This value will be computed later by the caller.
m.activityNameAndWeightedMountCount.value().mountCount = 0; // This value will be computed later by the caller.
+ // We will display the sleep flag only if it is not expired (15 minutes timeout, hardcoded).
+ // This allows having a single decision point instead of implementing is at the consumer levels.
+ if (rqSummary.sleepInfo && (::time(nullptr) < (rqSummary.sleepInfo.value().sleepStartTime
+ + (int64_t) rqSummary.sleepInfo.value().sleepTime)) ) {
+ m.sleepingMount = true;
+ m.sleepStartTime = rqSummary.sleepInfo.value().sleepStartTime;
+ m.diskSystemSleptFor = rqSummary.sleepInfo.value().diskSystemSleptFor;
+ m.sleepTime = rqSummary.sleepInfo.value().sleepTime;
+ }
}
}
if (jobsWithoutActivity) {
@@ -338,18 +347,27 @@ void OStoreDB::fetchMountInfo(SchedulerDatabase::TapeMountDecisionInfo& tmdi, Ro
auto & m = tmdi.potentialMounts.back();
m.vid = rqp.vid;
m.type = cta::common::dataStructures::MountType::Retrieve;
- m.bytesQueued = rqueue.getJobsSummary().bytes;
- m.filesQueued = rqueue.getJobsSummary().jobs;
- m.oldestJobStartTime = rqueue.getJobsSummary().oldestJobStartTime;
- m.priority = rqueue.getJobsSummary().priority;
- m.maxDrivesAllowed = rqueue.getJobsSummary().maxDrivesAllowed;
- m.minRequestAge = rqueue.getJobsSummary().minRetrieveRequestAge;
+ m.bytesQueued = rqSummary.bytes;
+ m.filesQueued = rqSummary.jobs;
+ m.oldestJobStartTime = rqSummary.oldestJobStartTime;
+ m.priority = rqSummary.priority;
+ m.maxDrivesAllowed = rqSummary.maxDrivesAllowed;
+ m.minRequestAge = rqSummary.minRetrieveRequestAge;
m.logicalLibrary = ""; // The logical library is not known here, and will be determined by the caller.
m.tapePool = ""; // The tape pool is not know and will be determined by the caller.
m.vendor = ""; // The vendor is not known here, and will be determined by the caller.
m.mediaType = ""; // The logical library is not known here, and will be determined by the caller.
m.vo = ""; // The vo is not known here, and will be determined by the caller.
m.capacityInBytes = 0; // The capacity is not known here, and will be determined by the caller.
+ // We will display the sleep flag only if it is not expired (15 minutes timeout, hardcoded).
+ // This allows having a single decision point instead of implementing is at the consumer levels.
+ if (rqSummary.sleepInfo && (::time(nullptr) < (rqSummary.sleepInfo.value().sleepStartTime
+ + (int64_t) rqSummary.sleepInfo.value().sleepTime)) ) {
+ m.sleepingMount = true;
+ m.sleepStartTime = rqSummary.sleepInfo.value().sleepStartTime;
+ m.diskSystemSleptFor = rqSummary.sleepInfo.value().diskSystemSleptFor;
+ rqSummary.sleepInfo.value().sleepTime;
+ }
}
} else {
tmdi.queueTrimRequired = true;
@@ -1102,7 +1120,7 @@ void OStoreDB::setRetrieveJobBatchReportedToUser(std::list<cta::SchedulerDatabas
insertedElements.emplace_back(CaRQF::InsertedElement{
&j.job->m_retrieveRequest, tf_it->copyNb, tf_it->fSeq, j.job->archiveFile.fileSize,
common::dataStructures::MountPolicy(), serializers::RetrieveJobStatus::RJS_Failed,
- j.job->m_activityDescription
+ j.job->m_activityDescription, j.job->m_diskSystemName
});
}
try {
@@ -1129,7 +1147,8 @@ std::list<SchedulerDatabase::RetrieveQueueStatistics> OStoreDB::getRetrieveQueue
// OStoreDB::queueRetrieve()
//------------------------------------------------------------------------------
SchedulerDatabase::RetrieveRequestInfo OStoreDB::queueRetrieve(cta::common::dataStructures::RetrieveRequest& rqst,
- const cta::common::dataStructures::RetrieveFileQueueCriteria& criteria, log::LogContext &logContext) {
+ const cta::common::dataStructures::RetrieveFileQueueCriteria& criteria, const optional<std::string> diskSystemName,
+ log::LogContext &logContext) {
assertAgentAddressSet();
auto mutexForHelgrind = cta::make_unique<cta::threading::Mutex>();
cta::threading::MutexLocker mlForHelgrind(*mutexForHelgrind);
@@ -1183,6 +1202,7 @@ SchedulerDatabase::RetrieveRequestInfo OStoreDB::queueRetrieve(cta::common::data
rReq->setRetrieveFileQueueCriteria(criteria);
rReq->setActivityIfNeeded(rqst, criteria);
rReq->setCreationTime(rqst.creationLog.time);
+ if (diskSystemName) rReq->setDiskSystemName(diskSystemName.value());
// Find the job corresponding to the vid (and check we indeed have one).
auto jobs = rReq->getJobs();
objectstore::RetrieveRequest::JobDump job;
@@ -2233,7 +2253,10 @@ void OStoreDB::RepackRequest::setLastExpandedFSeq(uint64_t fseq){
//------------------------------------------------------------------------------
// OStoreDB::RepackRequest::addSubrequests()
//------------------------------------------------------------------------------
-uint64_t OStoreDB::RepackRequest::addSubrequestsAndUpdateStats(std::list<Subrequest>& repackSubrequests, cta::common::dataStructures::ArchiveRoute::FullMap& archiveRoutesMap, uint64_t maxFSeqLowBound, const uint64_t maxAddedFSeq, const cta::SchedulerDatabase::RepackRequest::TotalStatsFiles &totalStatsFiles, log::LogContext& lc) {
+uint64_t OStoreDB::RepackRequest::addSubrequestsAndUpdateStats(std::list<Subrequest>& repackSubrequests,
+ cta::common::dataStructures::ArchiveRoute::FullMap& archiveRoutesMap, uint64_t maxFSeqLowBound,
+ const uint64_t maxAddedFSeq, const cta::SchedulerDatabase::RepackRequest::TotalStatsFiles &totalStatsFiles,
+ disk::DiskSystemList diskSystemList, log::LogContext& lc) {
// We need to prepare retrieve requests names and reference them, create them, enqueue them.
uint64_t nbRetrieveSubrequestsCreated = 0;
objectstore::ScopedExclusiveLock rrl (m_repackRequest);
@@ -2280,6 +2303,11 @@ uint64_t OStoreDB::RepackRequest::addSubrequestsAndUpdateStats(std::list<Subrequ
// dsrr.errorReportURL: We leave this bank as the reporting will be done to the repack request,
// stored in the repack info.
rr->setSchedulerRequest(schedReq);
+ // Add the disk system information if needed.
+ try {
+ auto dsName = diskSystemList.getDSNAme(schedReq.dstURL);
+ rr->setDiskSystemName(dsName);
+ } catch (std::out_of_range &) {}
// Set the repack info.
RetrieveRequest::RepackInfo rRRepackInfo;
try {
@@ -2459,11 +2487,12 @@ uint64_t OStoreDB::RepackRequest::addSubrequestsAndUpdateStats(std::list<Subrequ
lc.log(log::ERR, "In OStoreDB::RepackRequest::addSubRequests(), reported the failed creation of Retrieve Requests to the Repack request");
}
// We now have created the subrequests. Time to enqueue.
+ // TODO: the lock/fetch could be parallelized
{
objectstore::Sorter sorter(*m_oStoreDB.m_agentReference, m_oStoreDB.m_objectStore, m_oStoreDB.m_catalogue);
- std::list<std::unique_ptr<objectstore::ScopedExclusiveLock>> locks;
+ std::list<objectstore::ScopedExclusiveLock> locks;
for (auto &is: asyncInsertedSubrequestInfoList) {
- locks.push_back(cta::make_unique<objectstore::ScopedExclusiveLock>(*is.request));
+ locks.emplace_back(*is.request);
is.request->fetch();
sorter.insertRetrieveRequest(is.request, *m_oStoreDB.m_agentReference, is.activeCopyNb, lc);
}
@@ -2471,7 +2500,6 @@ uint64_t OStoreDB::RepackRequest::addSubrequestsAndUpdateStats(std::list<Subrequ
locks.clear();
sorter.flushAll(lc);
}
-
m_repackRequest.setLastExpandedFSeq(fSeq);
m_repackRequest.commit();
return nbRetrieveSubrequestsCreated;
@@ -2500,6 +2528,9 @@ void OStoreDB::RepackRequest::fail() {
m_repackRequest.commit();
}
+//------------------------------------------------------------------------------
+// OStoreDB::RepackRequest::requeueInToExpandQueue()
+//------------------------------------------------------------------------------
void OStoreDB::RepackRequest::requeueInToExpandQueue(log::LogContext& lc){
ScopedExclusiveLock rrl(m_repackRequest);
m_repackRequest.fetch();
@@ -2516,6 +2547,9 @@ void OStoreDB::RepackRequest::requeueInToExpandQueue(log::LogContext& lc){
rqteAlgo.referenceAndSwitchOwnership(nullopt, previousOwner, insertedElements, lc);
}
+//------------------------------------------------------------------------------
+// OStoreDB::RepackRequest::setExpandStartedAndChangeStatus()
+//------------------------------------------------------------------------------
void OStoreDB::RepackRequest::setExpandStartedAndChangeStatus(){
ScopedExclusiveLock rrl(m_repackRequest);
m_repackRequest.fetch();
@@ -2524,6 +2558,9 @@ void OStoreDB::RepackRequest::setExpandStartedAndChangeStatus(){
m_repackRequest.commit();
}
+//------------------------------------------------------------------------------
+// OStoreDB::RepackRequest::fillLastExpandedFSeqAndTotalStatsFile()
+//------------------------------------------------------------------------------
void OStoreDB::RepackRequest::fillLastExpandedFSeqAndTotalStatsFile(uint64_t& fSeq, TotalStatsFiles& totalStatsFiles) {
ScopedExclusiveLock rrl(m_repackRequest);
m_repackRequest.fetch();
@@ -2851,6 +2888,19 @@ void OStoreDB::updateDriveStatus(const common::dataStructures::DriveInfo& driveI
throw exception::Exception("Unexpected status in DriveRegister::reportDriveStatus");
}
ds.setState(driveState);
+ // If the drive is a state incompatible with space reservation, make sure there is none:
+ switch (inputs.status) {
+ case DriveStatus::CleaningUp:
+ case DriveStatus::Down:
+ case DriveStatus::Shutdown:
+ case DriveStatus::Unknown:
+ case DriveStatus::Unloading:
+ case DriveStatus::Unmounting:
+ case DriveStatus::Up:
+ ds.resetDiskSpaceReservation();
+ default:
+ break;
+ }
ds.commit();
}
@@ -3553,19 +3603,82 @@ const OStoreDB::RetrieveMount::MountInfo& OStoreDB::RetrieveMount::getMountInfo(
//------------------------------------------------------------------------------
// OStoreDB::RetrieveMount::getNextJobBatch()
//------------------------------------------------------------------------------
-std::list<std::unique_ptr<SchedulerDatabase::RetrieveJob>> OStoreDB::RetrieveMount::
-getNextJobBatch(uint64_t filesRequested, uint64_t bytesRequested, log::LogContext &logContext)
-{
- typedef objectstore::ContainerAlgorithms<RetrieveQueue,RetrieveQueueToTransferForUser> RQAlgos;
+std::list<std::unique_ptr<SchedulerDatabase::RetrieveJob>> OStoreDB::RetrieveMount::getNextJobBatch(uint64_t filesRequested,
+ uint64_t bytesRequested, cta::disk::DiskSystemFreeSpaceList & diskSystemFreeSpace, log::LogContext& logContext) {
+ // Pop a batch of files to retrieve and, for the ones having a documented disk system name, reserve the space
+ // that they will require. In case we cannot allocate the space for some of them, mark the destination filesystem as
+ // full and stop popping from it, after requeueing the jobs.
+ bool failedAllocation = false;
+ SchedulerDatabase::DiskSpaceReservationRequest diskSpaceReservationRequest;
+ typedef objectstore::ContainerAlgorithms<RetrieveQueue,RetrieveQueueToTransfer> RQAlgos;
RQAlgos rqAlgos(m_oStoreDB.m_objectStore, *m_oStoreDB.m_agentReference);
- RQAlgos::PopCriteria popCriteria(filesRequested, bytesRequested);
- auto jobs = rqAlgos.popNextBatch(mountInfo.vid, popCriteria, logContext);
- // We can construct the return value
+ RQAlgos::PoppedElementsBatch jobs;
+ retryPop:
+ {
+ RQAlgos::PopCriteria popCriteria(filesRequested, bytesRequested);
+ for (auto &dsts: m_diskSystemsToSkip) popCriteria.diskSystemsToSkip.insert({dsts.name, dsts.sleepTime});
+ jobs = rqAlgos.popNextBatch(mountInfo.vid, popCriteria, logContext);
+ // Try and allocate data for the popped jobs.
+ // Compute the necessary space in each targeted disk system.
+ std::map<std::string, uint64_t> spaceMap;
+ for (auto &j: jobs.elements)
+ if (j.diskSystemName)
+ diskSpaceReservationRequest.addRequest(j.diskSystemName.value(), j.archiveFile.fileSize);
+ // Get the existing reservation map from drives (including this drive's previous pending reservations).
+ auto previousDrivesReservations = getExistingDrivesReservations();
+ typedef std::pair<std::string, uint64_t> Res;
+ uint64_t previousDrivesReservationTotal = 0;
+ previousDrivesReservationTotal = std::accumulate(previousDrivesReservations.begin(), previousDrivesReservations.end(),
+ previousDrivesReservationTotal, [](uint64_t t, Res a){ return t+a.second;});
+ // Get the free space from disk systems involved.
+ std::set<std::string> diskSystemNames;
+ for (auto const & dsrr: diskSpaceReservationRequest) diskSystemNames.insert(dsrr.first);
+ try {
+ diskSystemFreeSpace.fetchDiskSystemFreeSpace(diskSystemNames, logContext);
+ } catch (std::exception &ex) {
+ // Leave a log message before letting the possible exception go up the stack.
+ log::ScopedParamContainer params(logContext);
+ params.add("exceptionWhat", ex.what());
+ logContext.log(log::ERR, "In OStoreDB::RetrieveMount::getNextJobBatch(): got an exception from diskSystemFreeSpace.fetchDiskSystemFreeSpace().");
+ throw;
+ }
+ // If any file system does not have enough space, mark it as full for this mount, requeue all (slight but rare inefficiency)
+ // and retry the pop.
+ for (auto const & ds: diskSystemNames) {
+ if (diskSystemFreeSpace.at(ds).freeSpace < diskSpaceReservationRequest.at(ds) + diskSystemFreeSpace.at(ds).targetedFreeSpace +
+ previousDrivesReservationTotal) {
+ m_diskSystemsToSkip.insert({ds, diskSystemFreeSpace.getDiskSystemList().at(ds).sleepTime});
+ failedAllocation = true;
+ log::ScopedParamContainer params(logContext);
+ params.add("diskSystemName", ds)
+ .add("freeSpace", diskSystemFreeSpace.at(ds).freeSpace)
+ .add("existingReservations", previousDrivesReservationTotal)
+ .add("spaceToReserve", diskSpaceReservationRequest.at(ds))
+ .add("targetedFreeSpace", diskSystemFreeSpace.at(ds).targetedFreeSpace);
+ logContext.log(log::WARNING, "In OStoreDB::RetrieveMount::getNextJobBatch(): could not allocate disk space for job batch.");
+ }
+ }
+ }
+ if (failedAllocation) {
+ std::list<std::unique_ptr<OStoreDB::RetrieveJob>> rjl;
+ for (auto & jle: jobs.elements) rjl.emplace_back(new OStoreDB::RetrieveJob(jle.retrieveRequest->getAddressIfSet(), m_oStoreDB, this));
+ requeueJobBatch(rjl, logContext);
+ rjl.clear();
+ // Clean up for the next round of popping
+ jobs.summary.files=0;
+ jobs.elements.clear();
+ failedAllocation = false;
+ diskSpaceReservationRequest.clear();
+ goto retryPop;
+ }
+ this->reserveDiskSpace(diskSpaceReservationRequest, logContext);
+ // Allocation went fine, we can construct the return value (we did not hit any full disk system.
std::list<std::unique_ptr<SchedulerDatabase::RetrieveJob>> ret;
for(auto &j : jobs.elements)
{
std::unique_ptr<OStoreDB::RetrieveJob> rj(new OStoreDB::RetrieveJob(j.retrieveRequest->getAddressIfSet(), m_oStoreDB, this));
rj->archiveFile = j.archiveFile;
+ rj->diskSystemName = j.diskSystemName;
rj->retrieveRequest = j.rr;
rj->selectedCopyNb = j.copyNb;
rj->isRepack = j.repackInfo.isRepack;
@@ -3577,6 +3690,85 @@ getNextJobBatch(uint64_t filesRequested, uint64_t bytesRequested, log::LogContex
return ret;
}
+//------------------------------------------------------------------------------
+// OStoreDB::RetrieveMount::requeueJobBatch()
+//------------------------------------------------------------------------------
+void OStoreDB::RetrieveMount::requeueJobBatch(std::list<std::unique_ptr<OStoreDB::RetrieveJob> >& jobBatch,
+ log::LogContext& logContext) {
+ objectstore::Sorter sorter(*m_oStoreDB.m_agentReference, m_oStoreDB.m_objectStore, m_oStoreDB.m_catalogue);
+ std::list<std::shared_ptr<objectstore::RetrieveRequest>> rrlist;
+ std::list<objectstore::ScopedExclusiveLock> locks;
+ for (auto & j: jobBatch) {
+ auto rr = std::make_shared<objectstore::RetrieveRequest>(j->m_retrieveRequest.getAddressIfSet(), m_oStoreDB.m_objectStore);
+ rrlist.push_back(rr);
+ locks.emplace_back(*rr);
+ rr->fetch();
+ sorter.insertRetrieveRequest(rr, *m_oStoreDB.m_agentReference, nullopt, logContext);
+ }
+ locks.clear();
+ rrlist.clear();
+ sorter.flushAll(logContext);
+}
+
+//------------------------------------------------------------------------------
+// OStoreDB::RetrieveMount::getExistingDrivesReservations()
+//------------------------------------------------------------------------------
+std::map<std::string, uint64_t> OStoreDB::RetrieveMount::getExistingDrivesReservations() {
+ objectstore::RootEntry re(m_oStoreDB.m_objectStore);
+ re.fetchNoLock();
+ objectstore::DriveRegister dr(re.getDriveRegisterAddress(), m_oStoreDB.m_objectStore);
+ dr.fetchNoLock();
+ auto driveAddresses = dr.getDriveAddresses();
+ std::list <objectstore::DriveState> dsList;
+ std::list <std::unique_ptr<objectstore::DriveState::AsyncLockfreeFetcher>> dsFetchers;
+ for (auto &d: driveAddresses) {
+ dsList.emplace_back(d.driveStateAddress, m_oStoreDB.m_objectStore);
+ dsFetchers.emplace_back(dsList.back().asyncLockfreeFetch());
+ }
+ auto dsf = dsFetchers.begin();
+ std::map<std::string, uint64_t> ret;
+ for (auto &d: dsList) {
+ try {
+ (*dsf)->wait();
+ dsf++;
+ for (auto &dsr: d.getDiskSpaceReservations()) {
+ try {
+ ret.at(dsr.first) += dsr.second;
+ } catch (std::out_of_range &) {
+ ret[dsr.first] = dsr.second;
+ }
+ }
+ } catch (objectstore::Backend::NoSuchObject) {
+ // If the drive status is not there, we just skip it.
+ dsf++;
+ }
+ }
+ return ret;
+}
+
+//------------------------------------------------------------------------------
+// OStoreDB::RetrieveMount::reserveDiskSpace()
+//------------------------------------------------------------------------------
+void OStoreDB::RetrieveMount::reserveDiskSpace(const DiskSpaceReservationRequest& diskSpaceReservation, log::LogContext & lc) {
+ // Try add our reservation to the drive status.
+ objectstore::DriveState ds(m_oStoreDB.m_objectStore);
+ objectstore::ScopedExclusiveLock dsl;
+ Helpers::getLockedAndFetchedDriveState(ds, dsl, *m_oStoreDB.m_agentReference, mountInfo.drive, lc, Helpers::CreateIfNeeded::doNotCreate);
+ for (auto const & dsr: diskSpaceReservation) ds.addDiskSpaceReservation(dsr.first, dsr.second);
+ ds.commit();
+}
+
+//------------------------------------------------------------------------------
+// OStoreDB::RetrieveMount::releaseDiskSpace()
+//------------------------------------------------------------------------------
+void OStoreDB::RetrieveMount::releaseDiskSpace(const DiskSpaceReservationRequest& diskSpaceReservation, log::LogContext & lc) {
+ // Try add our reservation to the drive status.
+ objectstore::DriveState ds(m_oStoreDB.m_objectStore);
+ objectstore::ScopedExclusiveLock dsl;
+ Helpers::getLockedAndFetchedDriveState(ds, dsl, *m_oStoreDB.m_agentReference, mountInfo.drive, lc, Helpers::CreateIfNeeded::doNotCreate);
+ for (auto const & dsr: diskSpaceReservation) ds.substractDiskSpaceReservation(dsr.first, dsr.second);
+ ds.commit();
+}
//------------------------------------------------------------------------------
// OStoreDB::RetrieveMount::complete()
//------------------------------------------------------------------------------
@@ -3668,10 +3860,12 @@ void OStoreDB::RetrieveMount::flushAsyncSuccessReports(std::list<cta::SchedulerD
std::map<std::string, std::list<OStoreDB::RetrieveJob*>> jobsToRequeueForRepackMap;
// We will wait on the asynchronously started reports of jobs, queue the retrieve jobs
// for report and remove them from ownership.
+ SchedulerDatabase::DiskSpaceReservationRequest diskSpaceReservationRequest;
// 1) Check the async update result.
common::dataStructures::MountPolicy mountPolicy;
for (auto & sDBJob: jobsBatch) {
auto osdbJob = castFromSchedDBJob(sDBJob);
+ if (osdbJob->diskSystemName) diskSpaceReservationRequest.addRequest(osdbJob->diskSystemName.value(), osdbJob->archiveFile.fileSize);
if (osdbJob->isRepack) {
try {
osdbJob->m_jobSucceedForRepackReporter->wait();
@@ -3730,6 +3924,7 @@ void OStoreDB::RetrieveMount::flushAsyncSuccessReports(std::list<cta::SchedulerD
}
}
}
+ releaseDiskSpace(diskSpaceReservationRequest, lc);
// 2) Queue the retrieve requests for repack.
for (auto & repackRequestQueue: jobsToRequeueForRepackMap) {
typedef objectstore::ContainerAlgorithms<RetrieveQueue,RetrieveQueueToReportToRepackForSuccess> RQTRTRFSAlgo;
@@ -3740,7 +3935,7 @@ void OStoreDB::RetrieveMount::flushAsyncSuccessReports(std::list<cta::SchedulerD
insertedRequests.push_back(RQTRTRFSAlgo::InsertedElement{&req->m_retrieveRequest, req->selectedCopyNb,
req->archiveFile.tapeFiles.at(req->selectedCopyNb).fSeq, req->archiveFile.fileSize,
mountPolicy,
- serializers::RetrieveJobStatus::RJS_ToReportToRepackForSuccess, req->m_activityDescription});
+ serializers::RetrieveJobStatus::RJS_ToReportToRepackForSuccess, req->m_activityDescription, req->m_diskSystemName});
requestToJobMap[&req->m_retrieveRequest] = req;
}
RQTRTRFSAlgo rQTRTRFSAlgo(m_oStoreDB.m_objectStore, *m_oStoreDB.m_agentReference);
@@ -4552,6 +4747,13 @@ void OStoreDB::RetrieveJob::failTransfer(const std::string &failureReason, log::
if (!m_jobOwned)
throw JobNotOwned("In OStoreDB::RetrieveJob::failTransfer: cannot fail a job not owned");
+ // Remove the space reservation for this job as we are done with it (if needed).
+ if (diskSystemName) {
+ SchedulerDatabase::DiskSpaceReservationRequest dsrr;
+ dsrr.addRequest(diskSystemName.value(), archiveFile.fileSize);
+ m_retrieveMount->releaseDiskSpace(dsrr, lc);
+ }
+
// Lock the retrieve request. Fail the job.
objectstore::ScopedExclusiveLock rel(m_retrieveRequest);
m_retrieveRequest.fetch();
@@ -4637,7 +4839,7 @@ void OStoreDB::RetrieveJob::failTransfer(const std::string &failureReason, log::
CaRqtr::InsertedElement::list insertedElements;
insertedElements.push_back(CaRqtr::InsertedElement{
&m_retrieveRequest, tf.copyNb, tf.fSeq, af.fileSize, rfqc.mountPolicy,
- serializers::RetrieveJobStatus::RJS_Failed, m_activityDescription
+ serializers::RetrieveJobStatus::RJS_Failed, m_activityDescription, m_diskSystemName
});
m_retrieveRequest.commit();
rel.release();
@@ -4667,7 +4869,7 @@ void OStoreDB::RetrieveJob::failTransfer(const std::string &failureReason, log::
}
case NextStep::EnqueueForTransferForUser: {
- typedef objectstore::ContainerAlgorithms<RetrieveQueue,RetrieveQueueToTransferForUser> CaRqtr;
+ typedef objectstore::ContainerAlgorithms<RetrieveQueue,RetrieveQueueToTransfer> CaRqtr;
// Algorithms suppose the objects are not locked
auto retryStatus = m_retrieveRequest.getRetryStatus(selectedCopyNb);
@@ -4676,7 +4878,7 @@ void OStoreDB::RetrieveJob::failTransfer(const std::string &failureReason, log::
std::set<std::string> candidateVids;
for(auto &tf : af.tapeFiles) {
- if(m_retrieveRequest.getJobStatus(tf.copyNb) == serializers::RetrieveJobStatus::RJS_ToTransferForUser)
+ if(m_retrieveRequest.getJobStatus(tf.copyNb) == serializers::RetrieveJobStatus::RJS_ToTransfer)
candidateVids.insert(tf.vid);
}
if(candidateVids.empty()) {
@@ -4704,8 +4906,8 @@ void OStoreDB::RetrieveJob::failTransfer(const std::string &failureReason, log::
CaRqtr::InsertedElement::list insertedElements;
insertedElements.push_back(CaRqtr::InsertedElement{
- &m_retrieveRequest, tf.copyNb, tf.fSeq, af.fileSize, rfqc.mountPolicy, serializers::RetrieveJobStatus::RJS_ToTransferForUser,
- m_activityDescription
+ &m_retrieveRequest, tf.copyNb, tf.fSeq, af.fileSize, rfqc.mountPolicy, serializers::RetrieveJobStatus::RJS_ToTransfer,
+ m_activityDescription, m_diskSystemName
});
CaRqtr caRqtr(m_oStoreDB.m_objectStore, *m_oStoreDB.m_agentReference);
@@ -4769,7 +4971,7 @@ void OStoreDB::RetrieveJob::failReport(const std::string &failureReason, log::Lo
CaRqtr::InsertedElement::list insertedElements;
insertedElements.push_back(CaRqtr::InsertedElement{
&m_retrieveRequest, tf.copyNb, tf.fSeq, af.fileSize, rfqc.mountPolicy,
- serializers::RetrieveJobStatus::RJS_ToReportToUserForFailure, m_activityDescription
+ serializers::RetrieveJobStatus::RJS_ToReportToUserForFailure, m_activityDescription, m_diskSystemName
});
caRqtr.referenceAndSwitchOwnership(tf.vid, insertedElements, lc);
log::ScopedParamContainer params(lc);
@@ -4788,7 +4990,7 @@ void OStoreDB::RetrieveJob::failReport(const std::string &failureReason, log::Lo
CaRqtr::InsertedElement::list insertedElements;
insertedElements.push_back(CaRqtr::InsertedElement{
&m_retrieveRequest, tf.copyNb, tf.fSeq, af.fileSize, rfqc.mountPolicy,
- serializers::RetrieveJobStatus::RJS_Failed, m_activityDescription
+ serializers::RetrieveJobStatus::RJS_Failed, m_activityDescription, m_diskSystemName
});
caRqtr.referenceAndSwitchOwnership(tf.vid, insertedElements, lc);
log::ScopedParamContainer params(lc);
diff --git a/scheduler/OStoreDB/OStoreDB.hpp b/scheduler/OStoreDB/OStoreDB.hpp
index fde33e1ddd93376e7a7092329c51f60a3d9add81..6ec6e4a37e4c203c628135ed2c753397a37b6f2b 100644
--- a/scheduler/OStoreDB/OStoreDB.hpp
+++ b/scheduler/OStoreDB/OStoreDB.hpp
@@ -217,7 +217,22 @@ public:
OStoreDB & m_oStoreDB;
public:
const MountInfo & getMountInfo() override;
- std::list<std::unique_ptr<SchedulerDatabase::RetrieveJob> > getNextJobBatch(uint64_t filesRequested, uint64_t bytesRequested, log::LogContext& logContext) override;
+ std::list<std::unique_ptr<cta::SchedulerDatabase::RetrieveJob> > getNextJobBatch(uint64_t filesRequested, uint64_t bytesRequested,
+ cta::disk::DiskSystemFreeSpaceList & diskSystemFreeSpace, log::LogContext& logContext) override;
+ private:
+ void requeueJobBatch(std::list<std::unique_ptr<OStoreDB::RetrieveJob> >& jobBatch,
+ log::LogContext& logContext);
+ std::map<std::string, uint64_t> getExistingDrivesReservations();
+ void reserveDiskSpace(const DiskSpaceReservationRequest& diskSpaceReservation, log::LogContext & lc);
+ struct DiskSystemToSkip {
+ std::string name;
+ uint64_t sleepTime;
+ bool operator<(const DiskSystemToSkip & o) const { return name < o.name; }
+ };
+ std::set<DiskSystemToSkip> m_diskSystemsToSkip;
+ public:
+ /// Public but non overriding function used by retrieve jobs (on failure to transfer):
+ void releaseDiskSpace(const DiskSpaceReservationRequest& diskSpaceReservation, log::LogContext & lc);
void complete(time_t completionTime) override;
void setDriveStatus(cta::common::dataStructures::DriveStatus status, time_t completionTime) override;
void setTapeSessionStats(const castor::tape::tapeserver::daemon::TapeSessionStats &stats) override;
@@ -258,6 +273,7 @@ public:
std::unique_ptr<objectstore::RetrieveRequest::AsyncJobSucceedForRepackReporter> m_jobSucceedForRepackReporter;
objectstore::RetrieveRequest::RepackInfo m_repackInfo;
optional<objectstore::RetrieveActivityDescription> m_activityDescription;
+ optional<std::string> m_diskSystemName;
};
static RetrieveJob * castFromSchedDBJob(SchedulerDatabase::RetrieveJob * job);
@@ -297,11 +313,11 @@ public:
CTA_GENERATE_EXCEPTION_CLASS(RetrieveRequestHasNoCopies);
CTA_GENERATE_EXCEPTION_CLASS(TapeCopyNumberOutOfRange);
SchedulerDatabase::RetrieveRequestInfo queueRetrieve(cta::common::dataStructures::RetrieveRequest& rqst,
- const cta::common::dataStructures::RetrieveFileQueueCriteria &criteria, log::LogContext &logContext) override;
+ const cta::common::dataStructures::RetrieveFileQueueCriteria &criteria, const optional<std::string> diskSystemName,
+ log::LogContext &logContext) override;
void cancelRetrieve(const std::string& instanceName, const cta::common::dataStructures::CancelRetrieveRequest& rqst,
log::LogContext& lc) override;
-
std::list<RetrieveRequestDump> getRetrieveRequestsByVid(const std::string& vid) const override;
std::list<RetrieveRequestDump> getRetrieveRequestsByRequester(const std::string& vid) const override;
@@ -348,7 +364,8 @@ public:
RepackRequest(const std::string &jobAddress, OStoreDB &oStoreDB) :
m_oStoreDB(oStoreDB), m_repackRequest(jobAddress, m_oStoreDB.m_objectStore){}
uint64_t addSubrequestsAndUpdateStats(std::list<Subrequest>& repackSubrequests, cta::common::dataStructures::ArchiveRoute::FullMap& archiveRoutesMap,
- uint64_t maxFSeqLowBound, const uint64_t maxAddedFSeq, const TotalStatsFiles &totalStatsFiles, log::LogContext& lc) override;
+ uint64_t maxFSeqLowBound, const uint64_t maxAddedFSeq, const TotalStatsFiles &totalStatsFiles, disk::DiskSystemList diskSystemList,
+ log::LogContext& lc) override;
void expandDone() override;
void fail() override;
void requeueInToExpandQueue(log::LogContext& lc) override;
diff --git a/scheduler/OStoreDB/OStoreDBFactory.hpp b/scheduler/OStoreDB/OStoreDBFactory.hpp
index 9802f13056575b38569a7ca9b7e253eacc1520fc..7d7b8b0d00f811cfb508c941302743cb2711cde3 100644
--- a/scheduler/OStoreDB/OStoreDBFactory.hpp
+++ b/scheduler/OStoreDB/OStoreDBFactory.hpp
@@ -219,8 +219,9 @@ public:
}
SchedulerDatabase::RetrieveRequestInfo queueRetrieve(common::dataStructures::RetrieveRequest& rqst,
- const common::dataStructures::RetrieveFileQueueCriteria &criteria, log::LogContext &logContext) override {
- return m_OStoreDB.queueRetrieve(rqst, criteria, logContext);
+ const common::dataStructures::RetrieveFileQueueCriteria &criteria, const optional<std::string> diskSystemName,
+ log::LogContext &logContext) override {
+ return m_OStoreDB.queueRetrieve(rqst, criteria, diskSystemName, logContext);
}
void cancelRetrieve(const std::string& instanceName, const cta::common::dataStructures::CancelRetrieveRequest& rqst,
diff --git a/scheduler/OStoreDB/OStoreDBTest.cpp b/scheduler/OStoreDB/OStoreDBTest.cpp
index 042245277ee7c79d2326cdbe81112320e9159163..4954c16b2dc4f205feb7a1b6c678e4a82c90e3de 100644
--- a/scheduler/OStoreDB/OStoreDBTest.cpp
+++ b/scheduler/OStoreDB/OStoreDBTest.cpp
@@ -223,7 +223,7 @@ TEST_P(OStoreDBTest, MemQueuesSharedAddToArchiveQueue) {
});
jobInsertions.emplace_back(std::async(std::launch::async ,lambdas.back()));
}
- for (auto &j: jobInsertions) { j.wait(); }
+ for (auto &j: jobInsertions) { j.get(); }
jobInsertions.clear();
lambdas.clear();
diff --git a/scheduler/RetrieveMount.cpp b/scheduler/RetrieveMount.cpp
index 470f7412c6ea2d535ea71df9fab2d6ed05f8d704..6ed0f8f5ec0356e2fa900057d1b840fda2496e9e 100644
--- a/scheduler/RetrieveMount.cpp
+++ b/scheduler/RetrieveMount.cpp
@@ -19,6 +19,7 @@
#include "scheduler/RetrieveMount.hpp"
#include "common/Timer.hpp"
#include "common/log/TimingList.hpp"
+#include "disk/DiskSystem.hpp"
//------------------------------------------------------------------------------
// constructor
@@ -111,7 +112,7 @@ std::string cta::RetrieveMount::getMediaType() const
}
//------------------------------------------------------------------------------
-// getVo()
+// getVendor()
//------------------------------------------------------------------------------
std::string cta::RetrieveMount::getVendor() const
{
@@ -122,6 +123,9 @@ std::string cta::RetrieveMount::getVendor() const
return sVendor.str();
}
+//------------------------------------------------------------------------------
+// getCapacityInBytes()
+//------------------------------------------------------------------------------
uint64_t cta::RetrieveMount::getCapacityInBytes() const {
if(!m_dbMount.get())
throw exception::Exception("In cta::RetrieveMount::getVendor(): got NULL dbMount");
@@ -135,9 +139,15 @@ std::list<std::unique_ptr<cta::RetrieveJob> > cta::RetrieveMount::getNextJobBatc
log::LogContext& logContext) {
if (!m_sessionRunning)
throw SessionNotRunning("In RetrieveMount::getNextJobBatch(): trying to get job from complete/not started session");
- // Try and get a new job from the DB
- std::list<std::unique_ptr<cta::SchedulerDatabase::RetrieveJob>> dbJobBatch(m_dbMount->getNextJobBatch(filesRequested,
- bytesRequested, logContext));
+ // Get the current file systems list from the catalogue
+ disk::DiskSystemList diskSystemList;
+ diskSystemList = m_catalogue.getAllDiskSystems();
+ // TODO: the diskSystemFreeSpaceList could be made a member of the retrieve mount and cache the fetched values, limiting the re-querying
+ // of the disk systems free space.
+ disk::DiskSystemFreeSpaceList diskSystemFreeSpaceList (diskSystemList);
+ // Try and get a new job from the DB. The DB mount (in memory object) is taking care of reserving the free space for the popped
+ // elements and query the disk systems, via the diskSystemFreeSpaceList object.
+ auto dbJobBatch = m_dbMount->getNextJobBatch(filesRequested, bytesRequested, diskSystemFreeSpaceList, logContext);
std::list<std::unique_ptr<RetrieveJob>> ret;
// We prepare the response
for (auto & sdrj: dbJobBatch) {
@@ -214,7 +224,7 @@ void cta::RetrieveMount::flushAsyncSuccessReports(std::queue<std::unique_ptr<cta
.add("diskFileId", job->archiveFile.diskFileId)
.add("lastKnownDiskPath", job->archiveFile.diskFileInfo.path);
}
- const std::string msg_error="In ArchiveMount::reportJobsBatchWritten(): got an standard exception";
+ const std::string msg_error="In RetrieveMount::reportJobsBatchWritten(): got an standard exception";
logContext.log(cta::log::ERR, msg_error);
// Failing here does not really affect the session so we can carry on. Reported jobs are reported, non-reported ones
// will be retried.
diff --git a/scheduler/RetrieveMount.hpp b/scheduler/RetrieveMount.hpp
index b8a5496777370801ea3040e57e8cf7616869f430..e5cb7b9028187436ff6ab4f596b0b6a5b2b1d06c 100644
--- a/scheduler/RetrieveMount.hpp
+++ b/scheduler/RetrieveMount.hpp
@@ -229,6 +229,11 @@ namespace cta {
/** An initialized-once factory for archive reports (indirectly used by ArchiveJobs) */
disk::DiskReporterFactory m_reporterFactory;
+ /**
+ * Internal tracking of the full disk systems. It is one strike out (for the mount duration).
+ */
+ std::set<std::string> m_fullDiskSystems;
+
/**
* A pointer to the file catalogue.
*/
diff --git a/scheduler/Scheduler.cpp b/scheduler/Scheduler.cpp
index 340e9c69241b22fb2da0819ab995cabe7561e6b9..8d108b2d1184f839fd81c75de06793f5938ce1e8 100644
--- a/scheduler/Scheduler.cpp
+++ b/scheduler/Scheduler.cpp
@@ -211,8 +211,14 @@ std::string Scheduler::queueRetrieve(
// Get the queue criteria
common::dataStructures::RetrieveFileQueueCriteria queueCriteria;
queueCriteria = m_catalogue.prepareToRetrieveFile(instanceName, request.archiveFileID, request.requester, request.activity, lc);
+ auto diskSystemList = m_catalogue.getAllDiskSystems();
auto catalogueTime = t.secs(cta::utils::Timer::resetCounter);
- auto requestInfo = m_db.queueRetrieve(request, queueCriteria, lc);
+ // Determine disk system for this request, if any.
+ optional<std::string> diskSystemName;
+ try {
+ diskSystemName = diskSystemList.getDSNAme(request.dstURL);
+ } catch (std::out_of_range&) {}
+ auto requestInfo = m_db.queueRetrieve(request, queueCriteria, diskSystemName, lc);
auto schedulerDbTime = t.secs();
log::ScopedParamContainer spc(lc);
spc.add("fileId", request.archiveFileID)
@@ -624,8 +630,10 @@ void Scheduler::expandRepackRequest(std::unique_ptr<RepackRequest>& repackReques
// We know that the fSeq processed on the tape are >= initial fSeq + filesCount - 1 (or fSeq - 1 as we counted).
// We pass this information to the db for recording in the repack request. This will allow restarting from the right
// value in case of crash.
+ auto diskSystemList = m_catalogue.getAllDiskSystems();
+ timingList.insertAndReset("getDisksystemsListTime",t);
try{
- nbRetrieveSubrequestsQueued = repackRequest->m_dbReq->addSubrequestsAndUpdateStats(retrieveSubrequests, archiveRoutesMap, fSeq, maxAddedFSeq, totalStatsFile, lc);
+ nbRetrieveSubrequestsQueued = repackRequest->m_dbReq->addSubrequestsAndUpdateStats(retrieveSubrequests, archiveRoutesMap, fSeq, maxAddedFSeq, totalStatsFile, diskSystemList, lc);
} catch(const cta::ExpandRepackRequestException& e){
deleteRepackBuffer(std::move(dir));
throw e;
@@ -958,12 +966,13 @@ void Scheduler::sortAndGetTapesForMountInfo(std::unique_ptr<SchedulerDatabase::T
// We can now filter out the potential mounts for which their mount criteria
// is not yet met, filter out the potential mounts for which the maximum mount
- // quota is already reached, and weight the remaining by how much of their quota
- // is reached
+ // quota is already reached, filter out the retrieve requests put to sleep for lack of disk space,
+ // and weight the remaining by how much of their quota is reached.
for (auto m = mountInfo->potentialMounts.begin(); m!= mountInfo->potentialMounts.end();) {
// Get summary data
uint32_t existingMounts = 0;
uint32_t activityMounts = 0;
+ bool sleepingMount = false;
try {
existingMounts = existingMountsSummary
.at(TapePoolMountPair(m->tapePool, common::dataStructures::getMountBasicType(m->type)))
@@ -986,7 +995,10 @@ void Scheduler::sortAndGetTapesForMountInfo(std::unique_ptr<SchedulerDatabase::T
mountPassesACriteria = true;
if (!effectiveExistingMounts && ((time(NULL) - m->oldestJobStartTime) > m->minRequestAge))
mountPassesACriteria = true;
- if (!mountPassesACriteria || existingMounts >= m->maxDrivesAllowed) {
+ if (m->sleepingMount) {
+ sleepingMount = true;
+ }
+ if (!mountPassesACriteria || existingMounts >= m->maxDrivesAllowed || sleepingMount) {
log::ScopedParamContainer params(lc);
params.add("tapePool", m->tapePool);
if ( m->type == common::dataStructures::MountType::Retrieve) {
@@ -1002,6 +1014,7 @@ void Scheduler::sortAndGetTapesForMountInfo(std::unique_ptr<SchedulerDatabase::T
.add("minArchiveRequestAge", m->minRequestAge)
.add("existingMounts", existingMounts)
.add("maxDrivesAllowed", m->maxDrivesAllowed);
+ if (sleepingMount) params.add("fullDiskSystem", m->diskSystemSleptFor);
lc.log(log::DEBUG, "In Scheduler::sortAndGetTapesForMountInfo(): Removing potential mount not passing criteria");
m = mountInfo->potentialMounts.erase(m);
} else {
@@ -1508,6 +1521,13 @@ std::list<common::dataStructures::QueueAndMountSummary> Scheduler::getQueuesAndM
summary.bytesQueued = pm.bytesQueued;
summary.filesQueued = pm.filesQueued;
summary.oldestJobAge = time(nullptr) - pm.oldestJobStartTime ;
+ if (pm.sleepingMount) {
+ common::dataStructures::QueueAndMountSummary::SleepForSpaceInfo sfsi;
+ sfsi.startTime = pm.sleepStartTime;
+ sfsi.diskSystemName = pm.diskSystemSleptFor;
+ sfsi.sleepTime = pm.sleepTime;
+ summary.sleepForSpaceInfo = sfsi;
+ }
break;
default:
break;
diff --git a/scheduler/SchedulerDatabase.cpp b/scheduler/SchedulerDatabase.cpp
index 4c257ef242e7f756b4507b6d67f9cde89aaa26a5..4c1415c4ee8bba2578495e2d6d385c7eae8a03a2 100644
--- a/scheduler/SchedulerDatabase.cpp
+++ b/scheduler/SchedulerDatabase.cpp
@@ -32,4 +32,13 @@ SchedulerDatabase::RepackRequestStatistics::RepackRequestStatistics() {
operator [](s) = 0;
}
+void SchedulerDatabase::DiskSpaceReservationRequest::addRequest(const std::string& diskSystemName, uint64_t size) {
+ try {
+ at(diskSystemName) += size;
+ } catch (std::out_of_range &) {
+ operator[](diskSystemName) = size;
+ }
+}
+
+
} //namespace cta
diff --git a/scheduler/SchedulerDatabase.hpp b/scheduler/SchedulerDatabase.hpp
index 1467274a02618d1ceff38a6f725669d0c0d59028..ee7ba1c4b41745b805f3a0e5adf94577d6ce5a75 100644
--- a/scheduler/SchedulerDatabase.hpp
+++ b/scheduler/SchedulerDatabase.hpp
@@ -34,6 +34,7 @@
#include "common/dataStructures/CancelRetrieveRequest.hpp"
#include "common/dataStructures/RepackInfo.hpp"
#include "common/dataStructures/SecurityIdentity.hpp"
+#include "disk/DiskSystem.hpp"
#include "common/remoteFS/RemotePathAndStatus.hpp"
#include "common/exception/Exception.hpp"
#include "common/log/LogContext.hpp"
@@ -269,6 +270,7 @@ public:
* @param rqst The request.
* @param criteria The criteria retrieved from the CTA catalogue to be used to
* decide how to quue the request.
+ * @param diskSystemName optional disk system name if the destination matches a declared one.
* @param logContext context allowing logging db operation
* @return the selected vid (mostly for logging)
*/
@@ -277,7 +279,8 @@ public:
std::string requestId;
};
virtual RetrieveRequestInfo queueRetrieve(cta::common::dataStructures::RetrieveRequest &rqst,
- const cta::common::dataStructures::RetrieveFileQueueCriteria &criteria, log::LogContext &logContext) = 0;
+ const cta::common::dataStructures::RetrieveFileQueueCriteria &criteria, const optional<std::string> diskSystemName,
+ log::LogContext &logContext) = 0;
virtual void cancelRetrieve(const std::string & instanceName, const cta::common::dataStructures::CancelRetrieveRequest &rqst,
log::LogContext & lc) = 0;
@@ -347,6 +350,12 @@ public:
/*============ Retrieve management: tape server side ======================*/
class RetrieveJob;
+
+ struct DiskSpaceReservationRequest: public std::map<std::string, uint64_t> {
+ void addRequest(const std::string &diskSystemName, uint64_t size);
+ };
+
+public:
class RetrieveMount {
public:
struct MountInfo {
@@ -364,7 +373,7 @@ public:
} mountInfo;
virtual const MountInfo & getMountInfo() = 0;
virtual std::list<std::unique_ptr<cta::SchedulerDatabase::RetrieveJob>> getNextJobBatch(uint64_t filesRequested,
- uint64_t bytesRequested, log::LogContext& logContext) = 0;
+ uint64_t bytesRequested, cta::disk::DiskSystemFreeSpaceList & diskSystemFreeSpace, log::LogContext& logContext) = 0;
virtual void complete(time_t completionTime) = 0;
virtual void setDriveStatus(common::dataStructures::DriveStatus status, time_t completionTime) = 0;
virtual void setTapeSessionStats(const castor::tape::tapeserver::daemon::TapeSessionStats &stats) = 0;
@@ -385,6 +394,7 @@ public:
} reportType;
cta::common::dataStructures::ArchiveFile archiveFile;
cta::common::dataStructures::RetrieveRequest retrieveRequest;
+ optional<std::string> diskSystemName;
uint32_t selectedCopyNb;
bool isRepack = false;
/** Set the job successful (async). Wait() and end of report happen in RetrieveMount::flushAsyncSuccessReports() */
@@ -463,9 +473,11 @@ public:
/**
* Add Retrieve subrequests to the repack request and update its statistics
* @return the number of retrieve subrequests queued
- */
+ */
virtual uint64_t addSubrequestsAndUpdateStats(std::list<Subrequest>& repackSubrequests,
- cta::common::dataStructures::ArchiveRoute::FullMap & archiveRoutesMap, uint64_t maxFSeqLowBound, const uint64_t maxAddedFSeq, const TotalStatsFiles &totalStatsFiles, log::LogContext & lc) = 0;
+ cta::common::dataStructures::ArchiveRoute::FullMap & archiveRoutesMap, uint64_t maxFSeqLowBound,
+ const uint64_t maxAddedFSeq, const TotalStatsFiles &totalStatsFiles, disk::DiskSystemList diskSystemList,
+ log::LogContext & lc) = 0;
virtual void expandDone() = 0;
virtual void fail() = 0;
virtual void requeueInToExpandQueue(log::LogContext &lc) = 0;
@@ -565,6 +577,10 @@ public:
std::string logicalLibrary; /**< The logical library (for a retrieve) */
double ratioOfMountQuotaUsed; /**< The [ 0.0, 1.0 ] ratio of existing
* mounts/quota (for faire share of mounts)*/
+ bool sleepingMount = false; /**< Is the mount being slept due to lack of disk space? */
+ time_t sleepStartTime = 0; /**< Start time for the sleeping for lack of disk space. */
+ std::string diskSystemSleptFor;/**< Name of (one of) the disk system(s) that could was too full to start more retrieves. */
+ uint64_t sleepTime = 0; /**< Length of time to be slept for for this disk system. */
uint32_t mountCount; /**< The number of mounts for this tape pool (which is the current "chargeable" entity for quotas. */
struct ActivityNameAndWeightedMountCount {
std::string activity;
diff --git a/scheduler/SchedulerDatabaseTest.cpp b/scheduler/SchedulerDatabaseTest.cpp
index 178dd76f62914cc351175e89e733509b5f1e0cd0..3fa680ab30fb02e38490d8441689e533eb7b6032 100644
--- a/scheduler/SchedulerDatabaseTest.cpp
+++ b/scheduler/SchedulerDatabaseTest.cpp
@@ -24,6 +24,7 @@
#include "OStoreDB/OStoreDBFactory.hpp"
#include "objectstore/BackendRados.hpp"
#include "common/log/DummyLogger.hpp"
+#include "common/range.hpp"
#include "common/make_unique.hpp"
#ifdef STDOUT_LOGGING
#include "common/log/StdoutLogger.hpp"
@@ -189,7 +190,7 @@ TEST_P(SchedulerDatabaseTest, createManyArchiveJobs) {
});
jobInsertions.emplace_back(std::async(std::launch::async,lambdas.back()));
}
- for (auto &j: jobInsertions) { j.wait(); }
+ for (auto &j: jobInsertions) { j.get(); }
jobInsertions.clear();
lambdas.clear();
db.waitSubthreadsComplete();
@@ -269,7 +270,7 @@ TEST_P(SchedulerDatabaseTest, createManyArchiveJobs) {
});
jobInsertions.emplace_back(std::async(std::launch::async,lambdas.back()));
}
- for (auto &j: jobInsertions) { j.wait(); }
+ for (auto &j: jobInsertions) { j.get(); }
jobInsertions.clear();
lambdas.clear();
@@ -307,6 +308,189 @@ TEST_P(SchedulerDatabaseTest, createManyArchiveJobs) {
moutInfo.reset(nullptr);
}
+
+TEST_P(SchedulerDatabaseTest, popRetrieveRequestsWithDisksytem) {
+ using namespace cta;
+#ifndef STDOUT_LOGGING
+ cta::log::DummyLogger dl("", "");
+#else
+ cta::log::StdoutLogger dl("", "");
+#endif
+ cta::log::LogContext lc(dl);
+
+ cta::SchedulerDatabase &db = getDb();
+
+ // Create the disk system list
+ cta::disk::DiskSystemList diskSystemList;
+ diskSystemList.push_back(cta::disk::DiskSystem{"ds-A", "$root://a.disk.system/", "constantFreeSpace:999999999999", 60, 10UL*1000*1000*1000,
+ 15*60, common::dataStructures::EntryLog(), common::dataStructures::EntryLog{},"No comment"});
+ diskSystemList.push_back(cta::disk::DiskSystem{"ds-B", "$root://b.disk.system/", "constantFreeSpace:999999999999", 60, 10UL*1000*1000*1000,
+ 15*60, common::dataStructures::EntryLog(), common::dataStructures::EntryLog{},"No comment"});
+ cta::disk::DiskSystemFreeSpaceList diskSystemFreeSpaceList(diskSystemList);
+
+ // Inject 10 retrieve jobs to the db.
+ const size_t filesToDo = 10;
+ std::list<std::future<void>> jobInsertions;
+ std::list<std::function<void()>> lambdas;
+ for (auto i: cta::range<size_t>(filesToDo)) {
+ lambdas.emplace_back(
+ [i,&db,&lc,diskSystemList](){
+ cta::common::dataStructures::RetrieveRequest rr;
+ cta::log::LogContext locallc=lc;
+ cta::common::dataStructures::RetrieveFileQueueCriteria rfqc;
+ rfqc.mountPolicy.name = "mountPolicy";
+ rfqc.mountPolicy.archivePriority = 1;
+ rfqc.mountPolicy.archiveMinRequestAge = 0;
+ rfqc.mountPolicy.retrievePriority = 1;
+ rfqc.mountPolicy.retrieveMinRequestAge = 0;
+ rfqc.mountPolicy.maxDrivesAllowed = 10;
+ rfqc.mountPolicy.creationLog = { "u", "h", time(nullptr)};
+ rfqc.mountPolicy.lastModificationLog = { "u", "h", time(nullptr)};
+ rfqc.mountPolicy.comment = "comment";
+ rfqc.archiveFile.fileSize = 1000;
+ rfqc.archiveFile.tapeFiles.push_back(cta::common::dataStructures::TapeFile());
+ rfqc.archiveFile.tapeFiles.back().fSeq = i;
+ rfqc.archiveFile.tapeFiles.back().vid = "vid";
+ rr.creationLog = { "user", "host", time(nullptr)};
+ uuid_t fileUUID;
+ uuid_generate(fileUUID);
+ char fileUUIDStr[37];
+ uuid_unparse(fileUUID, fileUUIDStr);
+ rr.diskFileInfo.path = std::string("/uuid/")+fileUUIDStr;
+ rr.requester = { "user", "group" };
+ rr.dstURL = std::string ("root://") + (i%2?"b":"a") + ".disk.system/" + std::to_string(i);
+ std::string dsName = (i%2?"ds-B":"ds-A");
+ db.queueRetrieve(rr, rfqc, dsName, locallc);
+ });
+ jobInsertions.emplace_back(std::async(std::launch::async,lambdas.back()));
+ }
+ for (auto &j: jobInsertions) { j.get(); }
+ jobInsertions.clear();
+ lambdas.clear();
+ db.waitSubthreadsComplete();
+
+ // Then load all archive jobs into memory
+ // Create mount.
+ auto moutInfo = db.getMountInfo(lc);
+ ASSERT_EQ(1, moutInfo->potentialMounts.size());
+ auto rm=moutInfo->createRetrieveMount("vid", "tapePool", "drive", "library", "host", "vo","mediaType", "vendor",123456789,time(nullptr), cta::nullopt);
+ auto rjb = rm->getNextJobBatch(20,20*1000,diskSystemFreeSpaceList, lc);
+ ASSERT_EQ(filesToDo, rjb.size());
+ std::list <cta::SchedulerDatabase::RetrieveJob*> jobBatch;
+ for (auto &rj: rjb) {
+ rj->asyncSetSuccessful();
+ ASSERT_TRUE((bool)rj->diskSystemName);
+ ASSERT_EQ(rj->archiveFile.tapeFiles.front().fSeq%2?"ds-B":"ds-A", rj->diskSystemName.value());
+ jobBatch.emplace_back(rj.get());
+ }
+ rm->flushAsyncSuccessReports(jobBatch, lc);
+ rjb.clear();
+ ASSERT_EQ(0, rm->getNextJobBatch(20,20*1000,diskSystemFreeSpaceList, lc).size());
+ rm->complete(time(nullptr));
+ rm.reset(nullptr);
+ moutInfo.reset(nullptr);
+}
+
+TEST_P(SchedulerDatabaseTest, popRetrieveRequestsWithBackpressure) {
+ using namespace cta;
+#ifndef STDOUT_LOGGING
+ cta::log::DummyLogger dl("", "");
+#else
+ cta::log::StdoutLogger dl("", "");
+#endif
+ cta::log::LogContext lc(dl);
+
+ cta::SchedulerDatabase &db = getDb();
+
+ // Create the disk system list
+ cta::disk::DiskSystemList diskSystemList;
+ // Files are 5 * 1000 byte per disk system. Make sure only half fits, preventing the pop for one of them.
+ // We should then be able to pop one bacth of for a single disk system, and get the queue slept.
+ diskSystemList.push_back(cta::disk::DiskSystem{"ds-A", "$root://a.disk.system/", "constantFreeSpace:6000", 60, 0UL,
+ 15*60, common::dataStructures::EntryLog(), common::dataStructures::EntryLog{},"No comment"});
+ diskSystemList.push_back(cta::disk::DiskSystem{"ds-B", "$root://b.disk.system/", "constantFreeSpace:7500", 60, 5000UL,
+ 15*60, common::dataStructures::EntryLog(), common::dataStructures::EntryLog{},"No comment"});
+ cta::disk::DiskSystemFreeSpaceList diskSystemFreeSpaceList(diskSystemList);
+
+
+ // Inject 10 retrieve jobs to the db.
+ const size_t filesToDo = 10;
+ std::atomic<size_t> aFiles(0);
+ std::atomic<size_t> bFiles(0);
+ std::list<std::future<void>> jobInsertions;
+ std::list<std::function<void()>> lambdas;
+ for (auto i: cta::range<size_t>(filesToDo)) {
+ lambdas.emplace_back(
+ [i,&db,&lc,&aFiles, &bFiles, diskSystemList](){
+ cta::common::dataStructures::RetrieveRequest rr;
+ cta::log::LogContext locallc=lc;
+ cta::common::dataStructures::RetrieveFileQueueCriteria rfqc;
+ rfqc.mountPolicy.name = "mountPolicy";
+ rfqc.mountPolicy.archivePriority = 1;
+ rfqc.mountPolicy.archiveMinRequestAge = 0;
+ rfqc.mountPolicy.retrievePriority = 1;
+ rfqc.mountPolicy.retrieveMinRequestAge = 0;
+ rfqc.mountPolicy.maxDrivesAllowed = 10;
+ rfqc.mountPolicy.creationLog = { "u", "h", time(nullptr)};
+ rfqc.mountPolicy.lastModificationLog = { "u", "h", time(nullptr)};
+ rfqc.mountPolicy.comment = "comment";
+ rfqc.archiveFile.fileSize = 1000;
+ rfqc.archiveFile.tapeFiles.push_back(cta::common::dataStructures::TapeFile());
+ rfqc.archiveFile.tapeFiles.back().fSeq = i;
+ rfqc.archiveFile.tapeFiles.back().vid = "vid";
+ rr.creationLog = { "user", "host", time(nullptr)};
+ uuid_t fileUUID;
+ uuid_generate(fileUUID);
+ char fileUUIDStr[37];
+ uuid_unparse(fileUUID, fileUUIDStr);
+ rr.diskFileInfo.path = std::string("/uuid/")+fileUUIDStr;
+ rr.requester = { "user", "group" };
+ std::string dsName;
+ if (i%2) {
+ rr.dstURL = std::string("root://b.disk.system/") + std::to_string(i);
+ dsName = "ds-B";
+ ++bFiles;
+ } else {
+ rr.dstURL = std::string("root://a.disk.system/") + std::to_string(i);
+ dsName = "ds-A";
+ ++aFiles;
+ }
+ db.queueRetrieve(rr, rfqc, dsName, locallc);
+ });
+ jobInsertions.emplace_back(std::async(std::launch::async,lambdas.back()));
+ }
+ for (auto &j: jobInsertions) { j.get(); }
+ jobInsertions.clear();
+ lambdas.clear();
+ db.waitSubthreadsComplete();
+
+ // Then load all archive jobs into memory
+ // Create mount.
+ auto moutInfo = db.getMountInfo(lc);
+ ASSERT_EQ(1, moutInfo->potentialMounts.size());
+ auto rm=moutInfo->createRetrieveMount("vid", "tapePool", "drive", "library", "host", "vo","mediaType", "vendor",123456789,time(nullptr), cta::nullopt);
+ auto rjb = rm->getNextJobBatch(20,20*1000,diskSystemFreeSpaceList, lc);
+ ASSERT_EQ(aFiles, rjb.size());
+ std::list <cta::SchedulerDatabase::RetrieveJob*> jobBatch;
+ for (auto &rj: rjb) {
+ rj->asyncSetSuccessful();
+ ASSERT_TRUE((bool)rj->diskSystemName);
+ ASSERT_EQ(rj->archiveFile.tapeFiles.front().fSeq%2?"ds-B":"ds-A", rj->diskSystemName.value());
+ jobBatch.emplace_back(rj.get());
+ }
+ rm->flushAsyncSuccessReports(jobBatch, lc);
+ rjb.clear();
+ ASSERT_EQ(0, rm->getNextJobBatch(20,20*1000,diskSystemFreeSpaceList, lc).size());
+ rm->complete(time(nullptr));
+ rm.reset(nullptr);
+ moutInfo.reset(nullptr);
+ auto mi = db.getMountInfoNoLock(lc);
+ ASSERT_EQ(1, mi->potentialMounts.size());
+ ASSERT_EQ(bFiles, mi->potentialMounts.begin()->filesQueued);
+ ASSERT_TRUE(mi->potentialMounts.begin()->sleepingMount);
+ ASSERT_EQ("ds-B", mi->potentialMounts.begin()->diskSystemSleptFor);
+}
+
#undef TEST_MOCK_DB
#ifdef TEST_MOCK_DB
static cta::MockSchedulerDatabaseFactory mockDbFactory;
diff --git a/scheduler/SchedulerTest.cpp b/scheduler/SchedulerTest.cpp
index d7af0f0dfe54cfa26d5a90615aef7cdae56529c8..0d35428ba4da9a1f735bab6af76b7e52c73509b1 100644
--- a/scheduler/SchedulerTest.cpp
+++ b/scheduler/SchedulerTest.cpp
@@ -1389,6 +1389,7 @@ TEST_P(SchedulerTest, expandRepackRequest) {
auto &schedulerDB = getSchedulerDB();
setupDefaultCatalogue();
+ catalogue.createDiskSystem({"user", "host"}, "diskSystem", "/public_dir/public_file", "constantFreeSpace:10", 10, 10L*1000*1000*1000, 15*60, "no comment");
#ifdef STDOUT_LOGGING
diff --git a/tapeserver/castor/tape/tapeserver/daemon/DiskWriteTaskTest.cpp b/tapeserver/castor/tape/tapeserver/daemon/DiskWriteTaskTest.cpp
index f7b3c8e2db5d0c41192bb41757439ac17384e824..a49fc4bb658bb0803814a71e91df8b7495143b13 100644
--- a/tapeserver/castor/tape/tapeserver/daemon/DiskWriteTaskTest.cpp
+++ b/tapeserver/castor/tape/tapeserver/daemon/DiskWriteTaskTest.cpp
@@ -42,7 +42,9 @@
namespace unitTests{
class TestingDatabaseRetrieveMount: public cta::SchedulerDatabase::RetrieveMount {
const MountInfo & getMountInfo() override { throw std::runtime_error("Not implemented"); }
- std::list<std::unique_ptr<cta::SchedulerDatabase::RetrieveJob> > getNextJobBatch(uint64_t filesRequested, uint64_t bytesRequested, cta::log::LogContext& logContext) override { throw std::runtime_error("Not implemented");}
+ std::list<std::unique_ptr<cta::SchedulerDatabase::RetrieveJob> > getNextJobBatch(uint64_t filesRequested, uint64_t bytesRequested,
+ cta::disk::DiskSystemFreeSpaceList & diskSystemFreeSpace, cta::log::LogContext& logContext) override { throw std::runtime_error("Not implemented");}
+
void complete(time_t completionTime) override { throw std::runtime_error("Not implemented"); }
void setDriveStatus(cta::common::dataStructures::DriveStatus status, time_t completionTime) override { throw std::runtime_error("Not implemented"); }
void setTapeSessionStats(const castor::tape::tapeserver::daemon::TapeSessionStats &stats) override { throw std::runtime_error("Not implemented"); }
diff --git a/tapeserver/castor/tape/tapeserver/daemon/DiskWriteThreadPoolTest.cpp b/tapeserver/castor/tape/tapeserver/daemon/DiskWriteThreadPoolTest.cpp
index b758bd62753da181d249f3a011fcb8b943acf9f0..6ec2d62b6bfdba5c24dca0400952257fc96577a7 100644
--- a/tapeserver/castor/tape/tapeserver/daemon/DiskWriteThreadPoolTest.cpp
+++ b/tapeserver/castor/tape/tapeserver/daemon/DiskWriteThreadPoolTest.cpp
@@ -37,7 +37,8 @@ namespace unitTests{
class TestingDatabaseRetrieveMount: public cta::SchedulerDatabase::RetrieveMount {
const MountInfo & getMountInfo() override { throw std::runtime_error("Not implemented"); }
- std::list<std::unique_ptr<cta::SchedulerDatabase::RetrieveJob> > getNextJobBatch(uint64_t filesRequested, uint64_t bytesRequested, cta::log::LogContext& logContext) override { throw std::runtime_error("Not implemented");}
+ std::list<std::unique_ptr<cta::SchedulerDatabase::RetrieveJob> > getNextJobBatch(uint64_t filesRequested, uint64_t bytesRequested, cta::disk::DiskSystemFreeSpaceList & diskSystemFreeSpace, cta::log::LogContext& logContext) override { throw std::runtime_error("Not implemented");}
+
void complete(time_t completionTime) override { throw std::runtime_error("Not implemented"); }
void setDriveStatus(cta::common::dataStructures::DriveStatus status, time_t completionTime) override { throw std::runtime_error("Not implemented"); }
void setTapeSessionStats(const castor::tape::tapeserver::daemon::TapeSessionStats &stats) override { throw std::runtime_error("Not implemented"); }
diff --git a/tapeserver/castor/tape/tapeserver/daemon/RecallTaskInjectorTest.cpp b/tapeserver/castor/tape/tapeserver/daemon/RecallTaskInjectorTest.cpp
index ba992cd264588eb1879fe975254900d6601dbe19..cbcea00193aaa3a43e618c48f52c2305c30e3487 100644
--- a/tapeserver/castor/tape/tapeserver/daemon/RecallTaskInjectorTest.cpp
+++ b/tapeserver/castor/tape/tapeserver/daemon/RecallTaskInjectorTest.cpp
@@ -131,7 +131,8 @@ namespace unitTests
class TestingDatabaseRetrieveMount: public cta::SchedulerDatabase::RetrieveMount {
const MountInfo & getMountInfo() override { throw std::runtime_error("Not implemented"); }
- std::list<std::unique_ptr<cta::SchedulerDatabase::RetrieveJob> > getNextJobBatch(uint64_t filesRequested, uint64_t bytesRequested, cta::log::LogContext& logContext) override { throw std::runtime_error("Not implemented");}
+ std::list<std::unique_ptr<cta::SchedulerDatabase::RetrieveJob> > getNextJobBatch(uint64_t filesRequested, uint64_t bytesRequested,
+ cta::disk::DiskSystemFreeSpaceList & diskSystemFreeSpace, cta::log::LogContext& logContext) override { throw std::runtime_error("Not implemented");}
void complete(time_t completionTime) override { throw std::runtime_error("Not implemented"); }
void setDriveStatus(cta::common::dataStructures::DriveStatus status, time_t completionTime) override { throw std::runtime_error("Not implemented"); }
void setTapeSessionStats(const castor::tape::tapeserver::daemon::TapeSessionStats &stats) override { throw std::runtime_error("Not implemented"); }
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index fdde0f00a130b539ac1114f22ef710140d46fc45..fd7308506026681fd3f92a69fda01c0818b81153 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -63,6 +63,7 @@ target_link_libraries(cta-rdbmsUnitTests
ctadbconfigconnunittests
ctadbconfigstmtunittests
ctardbmsunittests
+ ctadisk
gtest
pthread
${PROTOBUF3_LIBRARIES})
diff --git a/tests/postgresqlUnitTests.sh b/tests/postgresqlUnitTests.sh
new file mode 100755
index 0000000000000000000000000000000000000000..34a86c07af82ac433deb532069ec6f09b44e1d50
--- /dev/null
+++ b/tests/postgresqlUnitTests.sh
@@ -0,0 +1,43 @@
+#!/bin/bash
+
+# Utility script to run postgresql tests like in CI.
+
+if ! rpm -q --quiet rh-postgresql96-postgresql-server; then
+ echo "Please install the package rh-postgresql96-postgresql-server to run this script."
+ exit 1
+fi
+
+if [[ ! ( -x ./catalogue/cta-catalogue-schema-create && -x ./tests/cta-rdbmsUnitTests ) ]]; then
+ echo "Please run this script from a build tree of CTA."
+ exit 1
+fi
+
+if [[ $(id -u) != "0" ]]; then
+ echo "Please run this script as root (sudo)."
+ exit 1;
+fi
+
+POSTGRESQL_DATA_DIR=/usr/local/cta_test_pgsql_data
+POSTGRESQL_LOG_DIR=/var/log/postgres
+echo POSTGRESQL_DATA_DIR=${POSTGRESQL_DATA_DIR}
+echo POSTGRESQL_LOG_DIR=${POSTGRESQL_LOG_DIR}
+mkdir -p ${POSTGRESQL_DATA_DIR}
+chown -R postgres:postgres ${POSTGRESQL_DATA_DIR}
+mkdir -p ${POSTGRESQL_LOG_DIR}
+chown -R postgres:postgres ${POSTGRESQL_LOG_DIR}
+export LD_LIBRARY_PATH=/opt/rh/rh-postgresql96/root/usr/lib64
+POSTGRES_BIN=/opt/rh/rh-postgresql96/root/usr/bin
+echo POSTGRES_BIN=${POSTGRES_BIN}
+(cd / ;
+runuser -u postgres -- ${POSTGRES_BIN}/initdb -D ${POSTGRESQL_DATA_DIR}
+runuser -u postgres -- ${POSTGRES_BIN}/pg_ctl start -w -t 10 -D ${POSTGRESQL_DATA_DIR} -l ${POSTGRESQL_LOG_DIR}/cta_test_postgres.log
+runuser -u postgres -- ${POSTGRES_BIN}/createdb cta
+runuser -u postgres -- ${POSTGRES_BIN}/createuser -E cta
+)
+CTA_CATALOGUE_CONF=`mktemp`
+echo CTA_CATALOGUE_CONF=${CTA_CATALOGUE_CONF}
+echo 'postgresql:postgresql://cta@localhost/cta' > ${CTA_CATALOGUE_CONF}
+./catalogue/cta-catalogue-schema-create ${CTA_CATALOGUE_CONF}
+./tests/cta-rdbmsUnitTests ${CTA_CATALOGUE_CONF}
+(cd / ; runuser -u postgres -- ${POSTGRES_BIN}/pg_ctl stop -D ${POSTGRESQL_DATA_DIR})
+rm -rf ${POSTGRESQL_DATA_DIR}
\ No newline at end of file
diff --git a/xroot_plugins/XrdCtaDiskSystemLs.hpp b/xroot_plugins/XrdCtaDiskSystemLs.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..79644ee4c219ab12be32318aea7b99a2de3229df
--- /dev/null
+++ b/xroot_plugins/XrdCtaDiskSystemLs.hpp
@@ -0,0 +1,107 @@
+/*!
+ * @project The CERN Tape Archive (CTA)
+ * @brief CTA TapePool Ls stream implementation
+ * @copyright Copyright 2019 CERN
+ * @license This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <xroot_plugins/XrdCtaStream.hpp>
+#include <xroot_plugins/XrdSsiCtaRequestMessage.hpp>
+
+#include "disk/DiskSystem.hpp"
+
+
+namespace cta { namespace xrd {
+
+/*!
+ * Stream object which implements "disksystem ls" command
+ */
+class DiskSystemLsStream: public XrdCtaStream{
+public:
+ /*!
+ * Constructor
+ *
+ * @param[in] requestMsg RequestMessage containing command-line arguments
+ * @param[in] catalogue CTA Catalogue
+ * @param[in] scheduler CTA Scheduler
+ */
+ DiskSystemLsStream(const RequestMessage &requestMsg, cta::catalogue::Catalogue &catalogue, cta::Scheduler &scheduler);
+
+private:
+ /*!
+ * Can we close the stream?
+ */
+ virtual bool isDone() const {
+ return m_diskSystemList.empty();
+ }
+
+ /*!
+ * Fill the buffer
+ */
+ virtual int fillBuffer(XrdSsiPb::OStreamBuffer<Data> *streambuf);
+
+ cta::disk::DiskSystemList m_diskSystemList; //!< List of disk systems from the catalogue
+
+ static constexpr const char* const LOG_SUFFIX = "DiskSystemLsStream"; //!< Identifier for log messages
+};
+
+
+DiskSystemLsStream::DiskSystemLsStream(const RequestMessage &requestMsg, cta::catalogue::Catalogue &catalogue, cta::Scheduler &scheduler) :
+ XrdCtaStream(catalogue, scheduler),
+ m_diskSystemList(catalogue.getAllDiskSystems())
+{
+ using namespace cta::admin;
+
+ XrdSsiPb::Log::Msg(XrdSsiPb::Log::DEBUG, LOG_SUFFIX, "DiskSystemLsStream() constructor");
+}
+
+ std::string name;
+ std::string fileRegexp;
+ std::string freeSpaceQueryURL;
+ uint64_t refreshInterval;
+ uint64_t targetedFreeSpace;
+
+ cta::common::dataStructures::EntryLog creationLog;
+ cta::common::dataStructures::EntryLog lastModificationLog;
+ std::string comment;
+
+int DiskSystemLsStream::fillBuffer(XrdSsiPb::OStreamBuffer<Data> *streambuf) {
+ for(bool is_buffer_full = false; !m_diskSystemList.empty() && !is_buffer_full; m_diskSystemList.pop_front()) {
+ Data record;
+
+ auto &ds = m_diskSystemList.front();
+ auto ds_item = record.mutable_dsls_item();
+
+ ds_item->set_name(ds.name);
+ ds_item->set_file_regexp(ds.fileRegexp);
+ ds_item->set_free_space_query_url(ds.freeSpaceQueryURL);
+ ds_item->set_refresh_interval(ds.refreshInterval);
+ ds_item->set_targeted_free_space(ds.targetedFreeSpace);
+ ds_item->set_sleep_time(ds.sleepTime);
+ ds_item->mutable_creation_log()->set_username(ds.creationLog.username);
+ ds_item->mutable_creation_log()->set_host(ds.creationLog.host);
+ ds_item->mutable_creation_log()->set_time(ds.creationLog.time);
+ ds_item->mutable_last_modification_log()->set_username(ds.lastModificationLog.username);
+ ds_item->mutable_last_modification_log()->set_host(ds.lastModificationLog.host);
+ ds_item->mutable_last_modification_log()->set_time(ds.lastModificationLog.time);
+ ds_item->set_comment(ds.comment);
+
+ is_buffer_full = streambuf->Push(record);
+ }
+ return streambuf->Size();
+}
+
+}} // namespace cta::xrd
diff --git a/xroot_plugins/XrdCtaShowQueues.hpp b/xroot_plugins/XrdCtaShowQueues.hpp
index 72bfd88cabe01439be4705414fa8277085350b23..f2a86590bff844ca74d135124b0440c931463192 100644
--- a/xroot_plugins/XrdCtaShowQueues.hpp
+++ b/xroot_plugins/XrdCtaShowQueues.hpp
@@ -112,6 +112,13 @@ int ShowQueuesStream::fillBuffer(XrdSsiPb::OStreamBuffer<Data> *streambuf) {
sq_item->set_disabled_tapes(sq.disabledTapes);
sq_item->set_rdonly_tapes(sq.readOnlyTapes);
sq_item->set_writable_tapes(sq.writableTapes);
+ if (sq.sleepForSpaceInfo) {
+ sq_item->set_sleeping_for_space(true);
+ sq_item->set_sleep_start_time(sq.sleepForSpaceInfo.value().startTime);
+ sq_item->set_disk_system_slept_for(sq.sleepForSpaceInfo.value().diskSystemName);
+ } else {
+ sq_item->set_sleeping_for_space(false);
+ }
is_buffer_full = streambuf->Push(record);
}
diff --git a/xroot_plugins/XrdSsiCtaRequestMessage.cpp b/xroot_plugins/XrdSsiCtaRequestMessage.cpp
index f7b4454675acf69134aa213cdabd38a2f443e6c1..0e194a60e3f032c1598c49a07dc06bb26dea6c0b 100644
--- a/xroot_plugins/XrdSsiCtaRequestMessage.cpp
+++ b/xroot_plugins/XrdSsiCtaRequestMessage.cpp
@@ -36,6 +36,7 @@ using XrdSsiPb::PbException;
#include "XrdCtaTapeLs.hpp"
#include "XrdCtaStorageClassLs.hpp"
#include "XrdCtaTapePoolLs.hpp"
+#include "XrdCtaDiskSystemLs.hpp"
namespace cta {
namespace xrd {
@@ -230,7 +231,19 @@ void RequestMessage::process(const cta::xrd::Request &request, cta::xrd::Respons
case cmd_pair(AdminCmd::CMD_TAPEPOOL, AdminCmd::SUBCMD_LS):
processTapePool_Ls(response, stream);
break;
-
+ case cmd_pair(AdminCmd::CMD_DISKSYSTEM, AdminCmd::SUBCMD_LS):
+ processDiskSystem_Ls(response, stream);
+ break;
+ case cmd_pair(AdminCmd::CMD_DISKSYSTEM, AdminCmd::SUBCMD_ADD):
+ processDiskSystem_Add(response);
+ break;
+ case cmd_pair(AdminCmd::CMD_DISKSYSTEM, AdminCmd::SUBCMD_RM):
+ processDiskSystem_Rm(response);
+ break;
+ case cmd_pair(AdminCmd::CMD_DISKSYSTEM, AdminCmd::SUBCMD_CH):
+ processDiskSystem_Ch(response);
+ break;
+
default:
throw PbException("Admin command pair <" +
AdminCmd_Cmd_Name(request.admincmd().cmd()) + ", " +
@@ -1523,6 +1536,80 @@ void RequestMessage::processTapePool_Ls(cta::xrd::Response &response, XrdSsiStre
+void RequestMessage::processDiskSystem_Ls(cta::xrd::Response &response, XrdSsiStream* &stream)
+{
+ using namespace cta::admin;
+
+ // Create a XrdSsi stream object to return the results
+ stream = new DiskSystemLsStream(*this, m_catalogue, m_scheduler);
+
+ response.set_show_header(HeaderType::DISKSYSTEM_LS);
+ response.set_type(cta::xrd::Response::RSP_SUCCESS);
+}
+
+void RequestMessage::processDiskSystem_Add(cta::xrd::Response &response)
+{
+ using namespace cta::admin;
+
+ const auto &name = getRequired(OptionString::DISK_SYSTEM);
+ const auto &fileRegexp = getRequired(OptionString::FILE_REGEXP);
+ const auto &freeSpaceQueryURL = getRequired(OptionString::FREE_SPACE_QUERY_URL);
+ const auto &refreshInterval = getRequired(OptionUInt64::REFRESH_INTERVAL);
+ const auto &targetedFreeSpace = getRequired(OptionUInt64::TARGETED_FREE_SPACE);
+ const auto &sleepTime = getRequired(OptionUInt64::SLEEP_TIME);
+ const auto &comment = getRequired(OptionString::COMMENT);
+
+ m_catalogue.createDiskSystem(m_cliIdentity, name, fileRegexp, freeSpaceQueryURL,
+ refreshInterval, targetedFreeSpace, sleepTime, comment);
+
+ response.set_type(cta::xrd::Response::RSP_SUCCESS);
+}
+
+void RequestMessage::processDiskSystem_Ch(cta::xrd::Response &response)
+{
+ using namespace cta::admin;
+
+ const auto &name = getRequired(OptionString::DISK_SYSTEM);
+ const auto &fileRegexp = getOptional(OptionString::FILE_REGEXP);
+ const auto &freeSpaceQueryURL = getOptional(OptionString::FREE_SPACE_QUERY_URL);
+ const auto refreshInterval = getOptional(OptionUInt64::REFRESH_INTERVAL);
+ const auto targetedFreeSpace = getOptional(OptionUInt64::TARGETED_FREE_SPACE);
+ const auto sleepTime = getOptional(OptionUInt64::SLEEP_TIME);
+ const auto comment = getOptional(OptionString::COMMENT);
+
+ if(comment) {
+ m_catalogue.modifyDiskSystemComment(m_cliIdentity, name, comment.value());
+ }
+ if(fileRegexp) {
+ m_catalogue.modifyDiskSystemFileRegexp(m_cliIdentity, name, fileRegexp.value());
+ }
+ if(freeSpaceQueryURL) {
+ m_catalogue.modifyDiskSystemFreeSpaceQueryURL(m_cliIdentity, name, freeSpaceQueryURL.value());
+ }
+ if (sleepTime) {
+ m_catalogue.modifyDiskSystemSleepTime(m_cliIdentity, name, sleepTime.value());
+ }
+ if(refreshInterval) {
+ m_catalogue.modifyDiskSystemRefreshInterval(m_cliIdentity, name, refreshInterval.value());
+ }
+ if(targetedFreeSpace) {
+ m_catalogue.modifyDiskSystemTargetedFreeSpace(m_cliIdentity, name, targetedFreeSpace.value());
+ }
+
+ response.set_type(cta::xrd::Response::RSP_SUCCESS);
+}
+
+void RequestMessage::processDiskSystem_Rm(cta::xrd::Response &response)
+{
+ using namespace cta::admin;
+
+ const auto &name = getRequired(OptionString::DISK_SYSTEM);
+
+ m_catalogue.deleteDiskSystem(name);
+
+ response.set_type(cta::xrd::Response::RSP_SUCCESS);
+}
+
std::string RequestMessage::setDriveState(const std::string ®ex, DriveState drive_state)
{
using namespace cta::admin;
diff --git a/xroot_plugins/XrdSsiCtaRequestMessage.hpp b/xroot_plugins/XrdSsiCtaRequestMessage.hpp
index 89eb5e68a24ce177723d3b5508720ec0bb270726..fb4774aeefa69221f73cd978c7d97c8e5fd6da96 100644
--- a/xroot_plugins/XrdSsiCtaRequestMessage.hpp
+++ b/xroot_plugins/XrdSsiCtaRequestMessage.hpp
@@ -185,7 +185,10 @@ private:
void processTapePool_Add (cta::xrd::Response &response);
void processTapePool_Ch (cta::xrd::Response &response);
void processTapePool_Rm (cta::xrd::Response &response);
-
+ void processDiskSystem_Add (cta::xrd::Response &response);
+ void processDiskSystem_Ch (cta::xrd::Response &response);
+ void processDiskSystem_Rm (cta::xrd::Response &response);
+
/*!
* Process AdminCmd events which can return a stream response
*
@@ -211,6 +214,7 @@ private:
admincmdstream_t processTapePool_Ls;
admincmdstream_t processTape_Ls;
admincmdstream_t processRepack_Ls;
+ admincmdstream_t processDiskSystem_Ls;
/*!
* Log an admin command