diff --git a/CMakeLists.txt b/CMakeLists.txt
index 05f083cb12253a1d7cb2f26736cae0b576fd277e..87f87a6f8889a3d744f049cfa2fafd0a3214be89 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -158,6 +158,7 @@ ELSE(DEFINED PackageOnly)
add_subdirectory(continuousintegration/orchestration/tests)
+ add_subdirectory(cta-release)
#Generate version information
configure_file(${PROJECT_SOURCE_DIR}/version.hpp.in
diff --git a/ReleaseNotes.md b/ReleaseNotes.md
index 2249afc82c8e87cc679fcffd891d83310bcdc07c..c9a48cd57bff2039da821ee5c64fe11f3bca137e 100644
--- a/ReleaseNotes.md
+++ b/ReleaseNotes.md
@@ -3,6 +3,24 @@
## Summary
### Features
+ - cta/CTA#1016 New options for filtering deleted files using `cta-admin rtf ls` command.
+ - cta/CTA#983 Add cta-release package for public binary rpm distribution.
+
+- [frontend] New command "tapefile rm" allows deleting a copy of a file from tape
+
+### Bug fixes
+
+ - cta/CTA#1014 Fix last column alignment when more than 1000 items are listed.
+
+# v4.0-5
+
+## Summary
+
+### Features
+- [frontend] Add options to "tapepool ls" to filter tapepools on their name, vo and encryption
+- cta/CTA#898 cta-send-event now gets the requester id and eos instance as command line arguments
+- cta/CTA#1005 "tape ls" now can filter tapes on wether they were imported from CASTOR
+- cta/CTA#1006 "repack ls" now shows the tapepool of the tape being repacked
### Bug fixes
@@ -58,7 +76,7 @@
- cta/CTA#777 Minimize mounts for dual copy tape pool recalls - Temporary fix
- cta/CTA#927 Bad message for archive route pointing non-existing pool
- cta/CTA#930 Batched the queueing and the deleting of the repack subrequests
-- cta/CTA#969 `xrdfs query prepare` malformed JSON output
+- cta/CTA#969 `xrdfs query prepare` malformed JSON output
- cta/CTA#972 Updates cta-taped and rmcd man pages
- cta/CTA#967 CI DB cleanup issues
- cta/CTA#982 cta-fst-gcd ignored eos files with fid>4294967295
@@ -81,7 +99,7 @@ CTA v3.1-14.
## Summary
-This version is a transition version between CTA v3.1-14 and CTA v4.0. The functionalities of the v4.0 are implemented,
+This version is a transition version between CTA v3.1-14 and CTA v4.0. The functionalities of the v4.0 are implemented,
but at the database level, the columns to be deleted have been put as NULLABLE and the NOT NULL columns to be added as NULLABLE.
The xrootd-ssi-protobuf-interface is not up-to-date with the CTA v4.0: deprecated fields have not been removed.
@@ -190,7 +208,7 @@ This release contains the CTA software Recommended Access Order (RAO) implemente
- Upgraded EOS to 4.8.24-1
- Upgraded xrootd to 4.12.5-1
- cta-admin repack ls tabular output improvements
-- Repack management execution can be disabled via the cta-taped configuration file
+- Repack management execution can be disabled via the cta-taped configuration file
- cta/CTA#907 Maintenance process can be disabled via the cta-taped configuration file
# Modifications
@@ -200,7 +218,7 @@ This release contains the CTA software Recommended Access Order (RAO) implemente
### Bug fixes
- cta/CTA#901 cta-admin tapefile ls too slow
-- cta/CTA#895 [catalogue] RdbmsCatalogue::deleteLogicalLibrary does not delete empty logical library
+- cta/CTA#895 [catalogue] RdbmsCatalogue::deleteLogicalLibrary does not delete empty logical library
- utils::trimString() now returns an empty string if the string passed in parameter contains only white-space characters
- Repack request and sub-requests are now unowned from their Agent when completed
@@ -413,7 +431,7 @@ command-line tool based on operator requests and removes the deprecated
## Summary
-### Features
+### Features
- New schema version 2.0
- *VIRTUAL_ORGANIZATION* has its own table
@@ -427,7 +445,7 @@ command-line tool based on operator requests and removes the deprecated
## Summary
-### Features
+### Features
- cta-admin tapefile ls command: list the tape files located in a specific tape
- New schema verification tool (cta-catalogue-schema-verify)
@@ -450,7 +468,7 @@ command-line tool based on operator requests and removes the deprecated
- The scheduler does not return a mount if a tape is disabled (unless if the tape is repacked with the --disabledtape flag set)
-### Improvements
+### Improvements
- CASTOR-To-CTA migration improvements
- Better cta-admin parameter checking and column formatting
diff --git a/catalogue/Catalogue.hpp b/catalogue/Catalogue.hpp
index 4a5893cbb66f2af5139e07faada5080015070d2d..2e8aa1358ff1f97413e57764165fcdf5546c6fdf 100644
--- a/catalogue/Catalogue.hpp
+++ b/catalogue/Catalogue.hpp
@@ -940,6 +940,36 @@ public:
virtual common::dataStructures::ArchiveFileSummary getTapeFileSummary(
const TapeFileSearchCriteria &searchCriteria = TapeFileSearchCriteria()) const = 0;
+ /**
+ * Deletes a tape file copy that meet the specified vid, copyNb and archiveFileId
+ * The copy must match all attributes, or an exception is thrown.
+ *
+ * @param vid The vid of the tape the file copy is stored on.
+ * @param archiveFileId The unique identifier of the archived file
+ * @param reason The reason for deleting the tape file copy
+ */
+ virtual void deleteTapeFileCopy(
+ const std::string &vid,
+ const uint64_t archiveFileId,
+ const std::string &reason) = 0;
+
+ /**
+ * Deletes a tape file copy that meet the specified vid, copyNb, diskFileId and diskInstanceName
+ * The copy must match all attributes, or an exception is thrown.
+ *
+ * @param vid The vid of the tape the file copy is stored on.
+ * @param diskFileId The identifier of the archive_file on disk
+ * @param diskInstanceName The name of the archived file disk instance
+ * @param reason The reason for deleting the tape file copy
+ *
+ */
+ virtual void deleteTapeFileCopy(
+ const std::string &vid,
+ const std::string &diskFileId,
+ const std::string &diskInstanceName,
+ const std::string &reason) = 0;
+
+
/**
* Returns the archive file with the specified unique identifier.
*
diff --git a/catalogue/CatalogueRetryWrapper.hpp b/catalogue/CatalogueRetryWrapper.hpp
index b0e0a67792b205325c7f8be6021e3855c175a30e..e64e0426359a75951485951810de76e8518f1e33 100644
--- a/catalogue/CatalogueRetryWrapper.hpp
+++ b/catalogue/CatalogueRetryWrapper.hpp
@@ -563,6 +563,14 @@ public:
return retryOnLostConnection(m_log, [&]{return m_catalogue->getTapeFileSummary(searchCriteria);}, m_maxTriesToConnect);
}
+ void deleteTapeFileCopy(const std::string &vid, const uint64_t archiveFileId, const std::string &reason) override {
+ return retryOnLostConnection(m_log, [&]{return m_catalogue->deleteTapeFileCopy(vid, archiveFileId, reason);}, m_maxTriesToConnect);
+ }
+
+ void deleteTapeFileCopy(const std::string &vid, const std::string &diskFileId, const std::string &diskInstanceName, const std::string &reason) override {
+ return retryOnLostConnection(m_log, [&]{return m_catalogue->deleteTapeFileCopy(vid, diskFileId, diskInstanceName, reason);}, m_maxTriesToConnect);
+ }
+
common::dataStructures::ArchiveFile getArchiveFileById(const uint64_t id) const override {
return retryOnLostConnection(m_log, [&]{return m_catalogue->getArchiveFileById(id);}, m_maxTriesToConnect);
}
diff --git a/catalogue/CatalogueTest.cpp b/catalogue/CatalogueTest.cpp
index 3ca691bf769e301488e5db35ce74b25cd441d5c8..bdd6bf47e1b2e0278fcc4250139353812d3b8340 100644
--- a/catalogue/CatalogueTest.cpp
+++ b/catalogue/CatalogueTest.cpp
@@ -15735,7 +15735,8 @@ TEST_P(cta_catalogue_CatalogueTest, filesArePutInTheFileRecycleLogInsteadOfBeing
//Check the diskFileId search criteria
std::string diskFileId = "12345678";
catalogue::RecycleTapeFileSearchCriteria criteria;
- criteria.diskFileId = diskFileId;
+ criteria.diskFileIds = std::vector<std::string>();
+ criteria.diskFileIds->push_back(diskFileId);
auto fileRecycleLogItor = m_catalogue->getFileRecycleLogItor(criteria);
ASSERT_TRUE(fileRecycleLogItor.hasMore());
auto fileRecycleLog = fileRecycleLogItor.next();
@@ -15746,10 +15747,80 @@ TEST_P(cta_catalogue_CatalogueTest, filesArePutInTheFileRecycleLogInsteadOfBeing
//Check the non existing diskFileId search criteria
std::string diskFileId = "DOES_NOT_EXIST";
catalogue::RecycleTapeFileSearchCriteria criteria;
- criteria.diskFileId = diskFileId;
+ criteria.diskFileIds = std::vector<std::string>();
+ criteria.diskFileIds->push_back(diskFileId);
auto fileRecycleLogItor = m_catalogue->getFileRecycleLogItor(criteria);
ASSERT_FALSE(fileRecycleLogItor.hasMore());
}
+ {
+ //Check the archiveID search criteria
+ uint64_t archiveFileId = 1;
+ catalogue::RecycleTapeFileSearchCriteria criteria;
+ criteria.archiveFileId = archiveFileId;
+ auto fileRecycleLogItor = m_catalogue->getFileRecycleLogItor(criteria);
+ ASSERT_TRUE(fileRecycleLogItor.hasMore());
+ auto fileRecycleLog = fileRecycleLogItor.next();
+ ASSERT_EQ(archiveFileId,fileRecycleLog.archiveFileId);
+ ASSERT_FALSE(fileRecycleLogItor.hasMore());
+ }
+ {
+ //Check the non existing archiveFileId search criteria
+ uint64_t archiveFileId = -1;
+ catalogue::RecycleTapeFileSearchCriteria criteria;
+ criteria.archiveFileId = archiveFileId;
+ auto fileRecycleLogItor = m_catalogue->getFileRecycleLogItor(criteria);
+ ASSERT_FALSE(fileRecycleLogItor.hasMore());
+ }
+ {
+ //Check the copynb search criteria
+ uint64_t copynb = 1;
+ catalogue::RecycleTapeFileSearchCriteria criteria;
+ criteria.copynb = copynb;
+ auto fileRecycleLogItor = m_catalogue->getFileRecycleLogItor(criteria);
+ int nbFileRecycleLogs = 0;
+ while(fileRecycleLogItor.hasMore()){
+ nbFileRecycleLogs++;
+ fileRecycleLogItor.next();
+ }
+ ASSERT_EQ(nbArchiveFiles,nbFileRecycleLogs);
+ }
+ {
+ //Check the disk instance search criteria
+ catalogue::RecycleTapeFileSearchCriteria criteria;
+ criteria.diskInstance = diskInstance;
+ auto fileRecycleLogItor = m_catalogue->getFileRecycleLogItor(criteria);
+ int nbFileRecycleLogs = 0;
+ while(fileRecycleLogItor.hasMore()){
+ nbFileRecycleLogs++;
+ fileRecycleLogItor.next();
+ }
+ ASSERT_EQ(nbArchiveFiles,nbFileRecycleLogs);
+ }
+ {
+ //Check multiple search criteria together
+ uint64_t copynb = 1;
+ uint64_t archiveFileId = 1;
+ std::string diskFileId = "12345678";
+ catalogue::RecycleTapeFileSearchCriteria criteria;
+ criteria.diskInstance = diskInstance;
+ criteria.copynb = copynb;
+ criteria.archiveFileId = archiveFileId;
+ criteria.diskFileIds = std::vector<std::string>();
+ criteria.diskFileIds->push_back(diskFileId);
+ criteria.vid = tape1.vid;
+
+ auto fileRecycleLogItor = m_catalogue->getFileRecycleLogItor(criteria);
+
+ ASSERT_TRUE(fileRecycleLogItor.hasMore());
+ auto fileRecycleLog = fileRecycleLogItor.next();
+ ASSERT_EQ(archiveFileId, fileRecycleLog.archiveFileId);
+ ASSERT_EQ(diskFileId, fileRecycleLog.diskFileId);
+ ASSERT_EQ(copynb, fileRecycleLog.copyNb);
+ ASSERT_EQ(tape1.vid, fileRecycleLog.vid);
+ ASSERT_EQ(diskInstance, fileRecycleLog.diskInstanceName);
+
+ ASSERT_FALSE(fileRecycleLogItor.hasMore());
+ }
}
TEST_P(cta_catalogue_CatalogueTest, sameFileWrittenToSameTapePutThePreviousCopyOnTheFileRecycleLog) {
@@ -16141,4 +16212,286 @@ TEST_P(cta_catalogue_CatalogueTest, modifyDriveConfig) {
m_catalogue->deleteDriveConfig(tapeDriveName, daemonUserName1.key());
}
+TEST_P(cta_catalogue_CatalogueTest, DeleteTapeFileCopyUsingArchiveID) {
+ using namespace cta;
+
+ const bool logicalLibraryIsDisabled= false;
+ const std::string tapePoolName1 = "tape_pool_name_1";
+ const std::string tapePoolName2 = "tape_pool_name_2";
+ const uint64_t nbPartialTapes = 1;
+ const bool isEncrypted = true;
+ const cta::optional<std::string> supply("value for the supply pool mechanism");
+ const std::string diskInstance = "disk_instance";
+ const std::string tapeDrive = "tape_drive";
+ const std::string reason = "reason";
+
+ m_catalogue->createMediaType(m_admin, m_mediaType);
+ m_catalogue->createLogicalLibrary(m_admin, m_tape1.logicalLibraryName, logicalLibraryIsDisabled, "Create logical library");
+ m_catalogue->createVirtualOrganization(m_admin, m_vo);
+ m_catalogue->createTapePool(m_admin, tapePoolName1, m_vo.name, nbPartialTapes, isEncrypted, supply, "Create tape pool");
+ m_catalogue->createTapePool(m_admin, tapePoolName2, m_vo.name, nbPartialTapes, isEncrypted, supply, "Create tape pool");
+ m_catalogue->createStorageClass(m_admin, m_storageClassDualCopy);
+
+ auto tape1 = m_tape1;
+ auto tape2 = m_tape2;
+ tape1.tapePoolName = tapePoolName1;
+ tape2.tapePoolName = tapePoolName2;
+
+ m_catalogue->createTape(m_admin, tape1);
+ m_catalogue->createTape(m_admin, tape2);
+
+ ASSERT_FALSE(m_catalogue->getArchiveFilesItor().hasMore());
+ const uint64_t archiveFileSize = 2 * 1000 * 1000 * 1000;
+
+
+ // Write a file on tape
+ {
+ std::set<catalogue::TapeItemWrittenPointer> tapeFilesWrittenCopy1;
+
+ std::ostringstream diskFileId;
+ diskFileId << 12345677;
+
+ std::ostringstream diskFilePath;
+ diskFilePath << "/test/file1";
+
+ auto fileWrittenUP=cta::make_unique<cta::catalogue::TapeFileWritten>();
+ auto & fileWritten = *fileWrittenUP;
+ fileWritten.archiveFileId = 1;
+ fileWritten.diskInstance = diskInstance;
+ fileWritten.diskFileId = diskFileId.str();
+ fileWritten.diskFilePath = diskFilePath.str();
+ fileWritten.diskFileOwnerUid = PUBLIC_DISK_USER;
+ fileWritten.diskFileGid = PUBLIC_DISK_GROUP;
+ fileWritten.size = archiveFileSize;
+ fileWritten.checksumBlob.insert(checksum::ADLER32, "1357");
+ fileWritten.storageClassName = m_storageClassDualCopy.name;
+ fileWritten.vid = tape1.vid;
+ fileWritten.fSeq = 1;
+ fileWritten.blockId = 1 * 100;
+ fileWritten.copyNb = 1;
+ fileWritten.tapeDrive = tapeDrive;
+ tapeFilesWrittenCopy1.emplace(fileWrittenUP.release());
+
+ m_catalogue->filesWrittenToTape(tapeFilesWrittenCopy1);
+ }
+
+ // Write a second copy of file on tape
+ {
+ std::set<catalogue::TapeItemWrittenPointer> tapeFilesWrittenCopy1;
+
+ std::ostringstream diskFileId;
+ diskFileId << 12345677;
+
+ std::ostringstream diskFilePath;
+ diskFilePath << "/test/file1";
+
+ auto fileWrittenUP=cta::make_unique<cta::catalogue::TapeFileWritten>();
+ auto & fileWritten = *fileWrittenUP;
+ fileWritten.archiveFileId = 1;
+ fileWritten.diskInstance = diskInstance;
+ fileWritten.diskFileId = diskFileId.str();
+ fileWritten.diskFilePath = diskFilePath.str();
+ fileWritten.diskFileOwnerUid = PUBLIC_DISK_USER;
+ fileWritten.diskFileGid = PUBLIC_DISK_GROUP;
+ fileWritten.size = archiveFileSize;
+ fileWritten.checksumBlob.insert(checksum::ADLER32, "1357");
+ fileWritten.storageClassName = m_storageClassDualCopy.name;
+ fileWritten.vid = tape2.vid;
+ fileWritten.fSeq = 1;
+ fileWritten.blockId = 1 * 100;
+ fileWritten.copyNb = 2;
+ fileWritten.tapeDrive = tapeDrive;
+ tapeFilesWrittenCopy1.emplace(fileWrittenUP.release());
+
+ m_catalogue->filesWrittenToTape(tapeFilesWrittenCopy1);
+ }
+ {
+ //Assert both copies written
+ auto archiveFile = m_catalogue->getArchiveFileById(1);
+ ASSERT_EQ(2, archiveFile.tapeFiles.size());
+ }
+
+ {
+ //delete copy of file on tape1
+ m_catalogue->deleteTapeFileCopy(tape1.vid, 1, reason);
+ auto archiveFile = m_catalogue->getArchiveFileById(1);
+ ASSERT_EQ(1, archiveFile.tapeFiles.size());
+ auto fileRecycleLogItor = m_catalogue->getFileRecycleLogItor();
+ ASSERT_TRUE(fileRecycleLogItor.hasMore());
+ //The deleted file (fSeq = 1) is on the recycle log
+ auto recycleFileLog = fileRecycleLogItor.next();
+ ASSERT_EQ(1, recycleFileLog.fSeq);
+ ASSERT_EQ(tape1.vid, recycleFileLog.vid);
+ ASSERT_EQ(1, recycleFileLog.archiveFileId);
+ ASSERT_EQ(1, recycleFileLog.copyNb);
+ ASSERT_EQ(1 * 100, recycleFileLog.blockId);
+ ASSERT_EQ("(Deleted using cta-admin tf rm) " + reason, recycleFileLog.reasonLog);
+ ASSERT_EQ(std::string("Not applicable for copies deleted with cta-admin tf rm"), recycleFileLog.diskFilePath.value());
+ }
+
+ {
+ //delete last copy of file should fail
+ ASSERT_THROW(m_catalogue->deleteTapeFileCopy(tape2.vid, 1, "reason"), exception::UserError);
+
+ }
+}
+
+TEST_P(cta_catalogue_CatalogueTest, DeleteTapeFileCopyDoesNotExist) {
+ using namespace cta;
+
+ const bool logicalLibraryIsDisabled= false;
+ const std::string tapePoolName1 = "tape_pool_name_1";
+ const std::string tapePoolName2 = "tape_pool_name_2";
+ const uint64_t nbPartialTapes = 1;
+ const bool isEncrypted = true;
+ const cta::optional<std::string> supply("value for the supply pool mechanism");
+ const std::string diskInstance = "disk_instance";
+ const std::string tapeDrive = "tape_drive";
+ const std::string reason = "reason";
+
+ m_catalogue->createMediaType(m_admin, m_mediaType);
+ m_catalogue->createLogicalLibrary(m_admin, m_tape1.logicalLibraryName, logicalLibraryIsDisabled, "Create logical library");
+ m_catalogue->createVirtualOrganization(m_admin, m_vo);
+ m_catalogue->createTapePool(m_admin, tapePoolName1, m_vo.name, nbPartialTapes, isEncrypted, supply, "Create tape pool");
+ m_catalogue->createTapePool(m_admin, tapePoolName2, m_vo.name, nbPartialTapes, isEncrypted, supply, "Create tape pool");
+ m_catalogue->createStorageClass(m_admin, m_storageClassDualCopy);
+
+ auto tape1 = m_tape1;
+ auto tape2 = m_tape2;
+ tape1.tapePoolName = tapePoolName1;
+ tape2.tapePoolName = tapePoolName2;
+
+ m_catalogue->createTape(m_admin, tape1);
+ m_catalogue->createTape(m_admin, tape2);
+
+ {
+ //delete copy of file that does not exist should fail
+ ASSERT_THROW(m_catalogue->deleteTapeFileCopy(tape2.vid, 1, reason), exception::UserError);
+
+ }
+}
+
+TEST_P(cta_catalogue_CatalogueTest, DeleteTapeFileCopyUsingFXID) {
+ using namespace cta;
+
+ const bool logicalLibraryIsDisabled= false;
+ const std::string tapePoolName1 = "tape_pool_name_1";
+ const std::string tapePoolName2 = "tape_pool_name_2";
+ const uint64_t nbPartialTapes = 1;
+ const bool isEncrypted = true;
+ const cta::optional<std::string> supply("value for the supply pool mechanism");
+ const std::string diskInstance = "disk_instance";
+ const std::string tapeDrive = "tape_drive";
+ const std::string reason = "reason";
+
+ m_catalogue->createMediaType(m_admin, m_mediaType);
+ m_catalogue->createLogicalLibrary(m_admin, m_tape1.logicalLibraryName, logicalLibraryIsDisabled, "Create logical library");
+ m_catalogue->createVirtualOrganization(m_admin, m_vo);
+ m_catalogue->createTapePool(m_admin, tapePoolName1, m_vo.name, nbPartialTapes, isEncrypted, supply, "Create tape pool");
+ m_catalogue->createTapePool(m_admin, tapePoolName2, m_vo.name, nbPartialTapes, isEncrypted, supply, "Create tape pool");
+ m_catalogue->createStorageClass(m_admin, m_storageClassDualCopy);
+
+ auto tape1 = m_tape1;
+ auto tape2 = m_tape2;
+ tape1.tapePoolName = tapePoolName1;
+ tape2.tapePoolName = tapePoolName2;
+
+ m_catalogue->createTape(m_admin, tape1);
+ m_catalogue->createTape(m_admin, tape2);
+
+ ASSERT_FALSE(m_catalogue->getArchiveFilesItor().hasMore());
+ const uint64_t archiveFileSize = 2 * 1000 * 1000 * 1000;
+
+
+ // Write a file on tape
+ {
+ std::set<catalogue::TapeItemWrittenPointer> tapeFilesWrittenCopy1;
+
+ std::ostringstream diskFileId;
+ diskFileId << 12345677;
+
+ std::ostringstream diskFilePath;
+ diskFilePath << "/test/file1";
+
+ auto fileWrittenUP=cta::make_unique<cta::catalogue::TapeFileWritten>();
+ auto & fileWritten = *fileWrittenUP;
+ fileWritten.archiveFileId = 1;
+ fileWritten.diskInstance = diskInstance;
+ fileWritten.diskFileId = diskFileId.str();
+ fileWritten.diskFilePath = diskFilePath.str();
+ fileWritten.diskFileOwnerUid = PUBLIC_DISK_USER;
+ fileWritten.diskFileGid = PUBLIC_DISK_GROUP;
+ fileWritten.size = archiveFileSize;
+ fileWritten.checksumBlob.insert(checksum::ADLER32, "1357");
+ fileWritten.storageClassName = m_storageClassDualCopy.name;
+ fileWritten.vid = tape1.vid;
+ fileWritten.fSeq = 1;
+ fileWritten.blockId = 1 * 100;
+ fileWritten.copyNb = 1;
+ fileWritten.tapeDrive = tapeDrive;
+ tapeFilesWrittenCopy1.emplace(fileWrittenUP.release());
+
+ m_catalogue->filesWrittenToTape(tapeFilesWrittenCopy1);
+ }
+
+ // Write a second copy of file on tape
+ {
+ std::set<catalogue::TapeItemWrittenPointer> tapeFilesWrittenCopy1;
+
+ std::ostringstream diskFileId;
+ diskFileId << 12345677;
+
+ std::ostringstream diskFilePath;
+ diskFilePath << "/test/file1";
+
+ auto fileWrittenUP=cta::make_unique<cta::catalogue::TapeFileWritten>();
+ auto & fileWritten = *fileWrittenUP;
+ fileWritten.archiveFileId = 1;
+ fileWritten.diskInstance = diskInstance;
+ fileWritten.diskFileId = diskFileId.str();
+ fileWritten.diskFilePath = diskFilePath.str();
+ fileWritten.diskFileOwnerUid = PUBLIC_DISK_USER;
+ fileWritten.diskFileGid = PUBLIC_DISK_GROUP;
+ fileWritten.size = archiveFileSize;
+ fileWritten.checksumBlob.insert(checksum::ADLER32, "1357");
+ fileWritten.storageClassName = m_storageClassDualCopy.name;
+ fileWritten.vid = tape2.vid;
+ fileWritten.fSeq = 1;
+ fileWritten.blockId = 1 * 100;
+ fileWritten.copyNb = 2;
+ fileWritten.tapeDrive = tapeDrive;
+ tapeFilesWrittenCopy1.emplace(fileWrittenUP.release());
+
+ m_catalogue->filesWrittenToTape(tapeFilesWrittenCopy1);
+ }
+ {
+ //Assert both copies written
+ auto archiveFile = m_catalogue->getArchiveFileById(1);
+ ASSERT_EQ(2, archiveFile.tapeFiles.size());
+ }
+
+ {
+ //delete copy of file on tape1
+ m_catalogue->deleteTapeFileCopy(tape1.vid, "BC614D", diskInstance, reason);
+ auto archiveFile = m_catalogue->getArchiveFileById(1);
+ ASSERT_EQ(1, archiveFile.tapeFiles.size());
+ auto fileRecycleLogItor = m_catalogue->getFileRecycleLogItor();
+ ASSERT_TRUE(fileRecycleLogItor.hasMore());
+ //The previous file (fSeq = 1) is on the recycle log
+ auto recycleFileLog = fileRecycleLogItor.next();
+ ASSERT_EQ(1, recycleFileLog.fSeq);
+ ASSERT_EQ(tape1.vid, recycleFileLog.vid);
+ ASSERT_EQ(1, recycleFileLog.archiveFileId);
+ ASSERT_EQ(1, recycleFileLog.copyNb);
+ ASSERT_EQ(1 * 100, recycleFileLog.blockId);
+ ASSERT_EQ("(Deleted using cta-admin tf rm) " + reason, recycleFileLog.reasonLog);
+ ASSERT_EQ(std::string("Not applicable for copies deleted with cta-admin tf rm"), recycleFileLog.diskFilePath.value());
+ }
+
+ {
+ //delete last copy of file should fail
+ ASSERT_THROW(m_catalogue->deleteTapeFileCopy(tape2.vid, "BC614D", diskInstance, reason), exception::UserError);
+ }
+}
+
} // namespace unitTests
diff --git a/catalogue/DummyCatalogue.hpp b/catalogue/DummyCatalogue.hpp
index fc47d051dc12763d4634a42bd128eb0153acb532..05a0a1a357e44bfcb854be426f077b8ecb0c9e27 100644
--- a/catalogue/DummyCatalogue.hpp
+++ b/catalogue/DummyCatalogue.hpp
@@ -103,6 +103,8 @@ public:
std::list<common::dataStructures::RequesterMountRule> getRequesterMountRules() const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
std::list<common::dataStructures::StorageClass> getStorageClasses() const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
common::dataStructures::ArchiveFileSummary getTapeFileSummary(const TapeFileSearchCriteria& searchCriteria) const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
+ void deleteTapeFileCopy(const std::string &vid, const uint64_t archiveFileId, const std::string &reason) override {throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented");}
+ void deleteTapeFileCopy(const std::string &vid, const std::string &diskFileId, const std::string &diskInstanceName, const std::string &reason) override {throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented");}
std::list<TapePool> getTapePools(const TapePoolSearchCriteria &searchCriteria) const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
cta::optional<TapePool> getTapePool(const std::string &tapePoolName) const override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
std::list<common::dataStructures::Tape> getTapes(const TapeSearchCriteria& searchCriteria) const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
diff --git a/catalogue/MysqlCatalogue.cpp b/catalogue/MysqlCatalogue.cpp
index 552bb129738fc641f0fd9894dec7dce33ca8d106..f9c184f415ac7979b9c60bae07289dbd074117c3 100644
--- a/catalogue/MysqlCatalogue.cpp
+++ b/catalogue/MysqlCatalogue.cpp
@@ -59,10 +59,10 @@ MysqlCatalogue::~MysqlCatalogue() {
//------------------------------------------------------------------------------
// createAndPopulateTempTableFxid
//------------------------------------------------------------------------------
-std::string MysqlCatalogue::createAndPopulateTempTableFxid(rdbms::Conn &conn, const TapeFileSearchCriteria &tapeFileSearchCriteria) const {
+std::string MysqlCatalogue::createAndPopulateTempTableFxid(rdbms::Conn &conn, const optional<std::vector<std::string>> &diskFileIds) const {
const std::string tempTableName = "TEMP_DISK_FXIDS";
- if(tapeFileSearchCriteria.diskFileIds) {
+ if(diskFileIds) {
try {
std::string sql = "CREATE TEMPORARY TABLE " + tempTableName + "(DISK_FILE_ID VARCHAR(100))";
try {
@@ -76,7 +76,7 @@ std::string MysqlCatalogue::createAndPopulateTempTableFxid(rdbms::Conn &conn, co
sql = "INSERT INTO " + tempTableName + " VALUES(:DISK_FILE_ID)";
auto stmt = conn.createStmt(sql);
- for(auto &diskFileId : tapeFileSearchCriteria.diskFileIds.value()) {
+ for(auto &diskFileId : diskFileIds.value()) {
stmt.bindString(":DISK_FILE_ID", diskFileId);
stmt.executeNonQuery();
}
@@ -785,5 +785,40 @@ void MysqlCatalogue::deleteTapeFilesAndArchiveFileFromRecycleBin(rdbms::Conn & c
}
}
+//------------------------------------------------------------------------------
+// copyTapeFileToFileRecyleLogAndDelete
+//------------------------------------------------------------------------------
+void MysqlCatalogue::copyTapeFileToFileRecyleLogAndDelete(rdbms::Conn & conn, const cta::common::dataStructures::ArchiveFile &file,
+ const std::string &reason, log::LogContext & lc) {
+ try {
+ utils::Timer t;
+ log::TimingList tl;
+ //We currently do an INSERT INTO and a DELETE FROM
+ //in a single transaction
+ conn.executeNonQuery("START TRANSACTION");
+ copyTapeFilesToFileRecycleLog(conn, file, reason);
+ tl.insertAndReset("insertToRecycleBinTime",t);
+ setTapeDirty(conn, file.archiveFileID);
+ tl.insertAndReset("setTapeDirtyTime",t);
+ deleteTapeFiles(conn,file);
+ tl.insertAndReset("deleteTapeFilesTime",t);
+ conn.commit();
+ tl.insertAndReset("commitTime",t);
+ log::ScopedParamContainer spc(lc);
+ spc.add("archiveFileId", file.archiveFileID);
+ spc.add("diskFileId", file.diskFileId);
+ spc.add("diskFilePath", file.diskFileInfo.path);
+ spc.add("diskInstance", file.diskInstance);
+ tl.addToLog(spc);
+ lc.log(log::INFO,"In MysqlCatalogue::copyArchiveFileToRecycleBinAndDelete: ArchiveFile moved to the recycle-bin.");
+
+ } catch(exception::UserError &) {
+ throw;
+ } catch(exception::Exception &ex) {
+ ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str());
+ throw;
+ }
+}
+
} // namespace catalogue
} // namespace cta
diff --git a/catalogue/MysqlCatalogue.hpp b/catalogue/MysqlCatalogue.hpp
index 1e6b178516ffd0d820f360981e39e882f5e69b07..74ed0b0f5efd921bb7e425d520b7ea0f8af0ab55 100644
--- a/catalogue/MysqlCatalogue.hpp
+++ b/catalogue/MysqlCatalogue.hpp
@@ -60,10 +60,10 @@ protected:
* Creates a temporary table from the list of disk file IDs provided in the search criteria.
*
* @param conn The database connection.
- * @param tapeFileSearchCriteria Search criteria containing a list of disk file IDs (fxid).
+ * @param diskFileIds List of disk file IDs (fxid).
* @return Name of the temporary table
*/
- std::string createAndPopulateTempTableFxid(rdbms::Conn &conn, const TapeFileSearchCriteria &tapeFileSearchCriteria) const override;
+ std::string createAndPopulateTempTableFxid(rdbms::Conn &conn, const optional<std::vector<std::string>> &diskFileIds) const override;
/**
* Returns a unique archive ID that can be used by a new archive file within
@@ -208,6 +208,18 @@ protected:
*/
void deleteTapeFilesAndArchiveFileFromRecycleBin(rdbms::Conn & conn, const uint64_t archiveFileId, log::LogContext & lc) override;
+ /**
+ * Copy the tape files from the TAPE_FILE tables to the FILE_RECYCLE_LOG table
+ * and deletes the TAPE_FILE entry.
+ * @param conn the database connection
+ * @param file the file to be deleted
+ * @param reason The reason for deleting the tape file copy
+ * @param lc the log context
+ */
+ void copyTapeFileToFileRecyleLogAndDelete(rdbms::Conn & conn, const cta::common::dataStructures::ArchiveFile &file,
+ const std::string &reason, log::LogContext & lc) override;
+
+
private:
/**
diff --git a/catalogue/OracleCatalogue.cpp b/catalogue/OracleCatalogue.cpp
index 08e63e5afd379df8d58b4a5ee7500276b4ffb9b4..e15b23d3d0f994cb26bc57a0c5f3364ddd859e56 100644
--- a/catalogue/OracleCatalogue.cpp
+++ b/catalogue/OracleCatalogue.cpp
@@ -154,11 +154,11 @@ OracleCatalogue::~OracleCatalogue() {
//------------------------------------------------------------------------------
// createAndPopulateTempTableFxid
//------------------------------------------------------------------------------
-std::string OracleCatalogue::createAndPopulateTempTableFxid(rdbms::Conn &conn, const TapeFileSearchCriteria &tapeFileSearchCriteria) const {
+std::string OracleCatalogue::createAndPopulateTempTableFxid(rdbms::Conn &conn, const optional<std::vector<std::string>> &diskFileIds) const {
const std::string tempTableName = "ORA$PTT_DISK_FXIDS";
try {
- if(tapeFileSearchCriteria.diskFileIds) {
+ if(diskFileIds) {
conn.setAutocommitMode(rdbms::AutocommitMode::AUTOCOMMIT_OFF);
std::string sql = "CREATE PRIVATE TEMPORARY TABLE " + tempTableName +
"(DISK_FILE_ID VARCHAR2(100))";
@@ -166,7 +166,7 @@ std::string OracleCatalogue::createAndPopulateTempTableFxid(rdbms::Conn &conn, c
sql = "INSERT INTO " + tempTableName + " VALUES(:DISK_FILE_ID)";
auto stmt = conn.createStmt(sql);
- for(auto &diskFileId : tapeFileSearchCriteria.diskFileIds.value()) {
+ for(auto &diskFileId : diskFileIds.value()) {
stmt.bindString(":DISK_FILE_ID", diskFileId);
stmt.executeNonQuery();
}
@@ -1114,6 +1114,40 @@ void OracleCatalogue::deleteTapeFilesAndArchiveFileFromRecycleBin(rdbms::Conn& c
}
}
+//------------------------------------------------------------------------------
+// copyTapeFileToFileRecyleLogAndDelete
+//------------------------------------------------------------------------------
+void OracleCatalogue::copyTapeFileToFileRecyleLogAndDelete(rdbms::Conn & conn, const cta::common::dataStructures::ArchiveFile &file, const std::string &reason, log::LogContext & lc) {
+ try {
+ utils::Timer t;
+ log::TimingList tl;
+ //We currently do an INSERT INTO and a DELETE FROM
+ //in a single transaction
+ conn.setAutocommitMode(rdbms::AutocommitMode::AUTOCOMMIT_OFF);
+ copyTapeFilesToFileRecycleLog(conn, file, reason);
+ tl.insertAndReset("insertToRecycleBinTime",t);
+ setTapeDirty(conn, file.archiveFileID);
+ tl.insertAndReset("setTapeDirtyTime",t);
+ deleteTapeFiles(conn,file);
+ tl.insertAndReset("deleteTapeFilesTime",t);
+ conn.setAutocommitMode(rdbms::AutocommitMode::AUTOCOMMIT_ON);
+ conn.commit();
+ tl.insertAndReset("commitTime",t);
+ log::ScopedParamContainer spc(lc);
+ spc.add("archiveFileId", file.archiveFileID);
+ spc.add("diskFileId", file.diskFileId);
+ spc.add("diskFilePath", file.diskFileInfo.path);
+ spc.add("diskInstance", file.diskInstance);
+ tl.addToLog(spc);
+ lc.log(log::INFO,"In OracleCatalogue::copyArchiveFileToRecycleBinAndDelete: ArchiveFile moved to the recycle-bin.");
+
+ } catch(exception::UserError &) {
+ throw;
+ } catch(exception::Exception &ex) {
+ ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str());
+ throw;
+ }
+}
} // namespace catalogue
diff --git a/catalogue/OracleCatalogue.hpp b/catalogue/OracleCatalogue.hpp
index 03067d1d544fea1e53e3e4ceccaaaa6bf9459a53..f0c05cf0176a027691eb1eb17f90958f59a36719 100644
--- a/catalogue/OracleCatalogue.hpp
+++ b/catalogue/OracleCatalogue.hpp
@@ -61,10 +61,10 @@ public:
* Creates a temporary table from the list of disk file IDs provided in the search criteria.
*
* @param conn The database connection.
- * @param tapeFileSearchCriteria Search criteria containing a list of disk file IDs (fxid).
+ * @param diskFileIds List of disk file IDs (fxid).
* @return Name of the temporary table
*/
- std::string createAndPopulateTempTableFxid(rdbms::Conn &conn, const TapeFileSearchCriteria &tapeFileSearchCriteria) const override;
+ std::string createAndPopulateTempTableFxid(rdbms::Conn &conn, const optional<std::vector<std::string>> &diskFileIds) const override;
/**
* Returns a unique archive ID that can be used by a new archive file within
@@ -251,6 +251,17 @@ private:
*/
void deleteTapeFilesAndArchiveFileFromRecycleBin(rdbms::Conn& conn, const uint64_t archiveFileId, log::LogContext& lc) override;
+ /**
+ * Copy the tape files from the TAPE_FILE tables to the FILE_RECYCLE_LOG table
+ * and deletes the TAPE_FILE entry.
+ * @param conn the database connection
+ * @param file the file to be deleted
+ * @param reason The reason for deleting the tape file copy
+ * @param lc the log context
+ */
+ void copyTapeFileToFileRecyleLogAndDelete(rdbms::Conn & conn, const cta::common::dataStructures::ArchiveFile &file,
+ const std::string &reason, log::LogContext & lc) override;
+
/**
* The size and checksum of a file.
*/
diff --git a/catalogue/PostgresCatalogue.cpp b/catalogue/PostgresCatalogue.cpp
index da343a4a58d41c7fbe436de4b1856f9a4ed7d32b..8967dd6901303673dbb3df209f735744e46e8534 100644
--- a/catalogue/PostgresCatalogue.cpp
+++ b/catalogue/PostgresCatalogue.cpp
@@ -152,10 +152,10 @@ PostgresCatalogue::~PostgresCatalogue() {
//------------------------------------------------------------------------------
// createAndPopulateTempTableFxid
//------------------------------------------------------------------------------
-std::string PostgresCatalogue::createAndPopulateTempTableFxid(rdbms::Conn &conn, const TapeFileSearchCriteria &tapeFileSearchCriteria) const {
+std::string PostgresCatalogue::createAndPopulateTempTableFxid(rdbms::Conn &conn, const optional<std::vector<std::string>> &diskFileIds) const {
const std::string tempTableName = "TEMP_DISK_FXIDS";
- if(tapeFileSearchCriteria.diskFileIds) {
+ if(diskFileIds) {
try {
std::string sql = "CREATE TEMPORARY TABLE " + tempTableName + "(DISK_FILE_ID VARCHAR(100))";
try {
@@ -169,7 +169,7 @@ std::string PostgresCatalogue::createAndPopulateTempTableFxid(rdbms::Conn &conn,
sql = "INSERT INTO " + tempTableName + " VALUES(:DISK_FILE_ID)";
auto stmt = conn.createStmt(sql);
- for(auto &diskFileId : tapeFileSearchCriteria.diskFileIds.value()) {
+ for(auto &diskFileId : diskFileIds.value()) {
stmt.bindString(":DISK_FILE_ID", diskFileId);
stmt.executeNonQuery();
}
@@ -1074,5 +1074,40 @@ void PostgresCatalogue::deleteTapeFilesAndArchiveFileFromRecycleBin(rdbms::Conn&
}
}
+//------------------------------------------------------------------------------
+// copyTapeFileToFileRecyleLogAndDelete
+//------------------------------------------------------------------------------
+void PostgresCatalogue::copyTapeFileToFileRecyleLogAndDelete(rdbms::Conn & conn, const cta::common::dataStructures::ArchiveFile &file,
+ const std::string &reason, log::LogContext & lc) {
+ try {
+ utils::Timer t;
+ log::TimingList tl;
+ //We currently do an INSERT INTO and a DELETE FROM
+ //in a single transaction
+ conn.executeNonQuery("BEGIN");
+ copyTapeFilesToFileRecycleLog(conn, file, reason);
+ tl.insertAndReset("insertToRecycleBinTime",t);
+ setTapeDirty(conn, file.archiveFileID);
+ tl.insertAndReset("setTapeDirtyTime",t);
+ deleteTapeFiles(conn,file);
+ tl.insertAndReset("deleteTapeFilesTime",t);
+ conn.commit();
+ tl.insertAndReset("commitTime",t);
+ log::ScopedParamContainer spc(lc);
+ spc.add("archiveFileId", file.archiveFileID);
+ spc.add("diskFileId", file.diskFileId);
+ spc.add("diskFilePath", file.diskFileInfo.path);
+ spc.add("diskInstance", file.diskInstance);
+ tl.addToLog(spc);
+ lc.log(log::INFO,"In PostgresCatalogue::copyArchiveFileToRecycleBinAndDelete: ArchiveFile moved to the recycle-bin.");
+
+ } catch(exception::UserError &) {
+ throw;
+ } catch(exception::Exception &ex) {
+ ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str());
+ throw;
+ }
+}
+
} // namespace catalogue
} // namespace cta
diff --git a/catalogue/PostgresCatalogue.hpp b/catalogue/PostgresCatalogue.hpp
index ee2c914b851487d39cd5cc307583aacdb1a65fa7..90bacb1e28ad5833af7c823b934e3423ed2a1cfe 100644
--- a/catalogue/PostgresCatalogue.hpp
+++ b/catalogue/PostgresCatalogue.hpp
@@ -88,10 +88,10 @@ public:
* Creates a temporary table from the list of disk file IDs provided in the search criteria.
*
* @param conn The database connection.
- * @param tapeFileSearchCriteria Search criteria containing a list of disk file IDs (fxid).
+ * @param diskFileIds List of disk file IDs (fxid).
* @return Name of the temporary table
*/
- std::string createAndPopulateTempTableFxid(rdbms::Conn &conn, const TapeFileSearchCriteria &tapeFileSearchCriteria) const override;
+ std::string createAndPopulateTempTableFxid(rdbms::Conn &conn, const optional<std::vector<std::string>> &diskFileIds) const override;
/**
* Returns a unique archive ID that can be used by a new archive file within
@@ -284,6 +284,17 @@ private:
* @param archiveFileId the archiveFileId of the file to delete from the recycle-bin
*/
virtual void deleteTapeFilesAndArchiveFileFromRecycleBin(rdbms::Conn & conn, const uint64_t archiveFileId, log::LogContext & lc);
+
+ /**
+ * Copy the tape files from the TAPE_FILE tables to the FILE_RECYCLE_LOG table
+ * and deletes the TAPE_FILE entry.
+ * @param conn the database connection
+ * @param file the file to be deleted
+ * @param reason The reason for deleting the tape file copy
+ * @param lc the log context
+ */
+ void copyTapeFileToFileRecyleLogAndDelete(rdbms::Conn & conn, const cta::common::dataStructures::ArchiveFile &file,
+ const std::string &reason, log::LogContext & lc) override;
}; // class PostgresCatalogue
diff --git a/catalogue/RdbmsCatalogue.cpp b/catalogue/RdbmsCatalogue.cpp
index 7f8ddd1c352133886331b33b8cde5cf661fbd724..1f3a640fe6b985a86120af6f55b48df91e1675bc 100644
--- a/catalogue/RdbmsCatalogue.cpp
+++ b/catalogue/RdbmsCatalogue.cpp
@@ -43,6 +43,7 @@
#include <string>
#include <time.h>
#include <tuple>
+#include <climits>
namespace cta {
namespace catalogue {
@@ -3882,7 +3883,7 @@ std::list<common::dataStructures::Tape> RdbmsCatalogue::getTapes(rdbms::Conn &co
sql += " TAPE.IS_FROM_CASTOR = :FROM_CASTOR";
addedAWhereConstraint = true;
}
-
+
sql += " ORDER BY TAPE.VID";
auto stmt = conn.createStmt(sql);
@@ -6973,7 +6974,7 @@ Catalogue::ArchiveFileItor RdbmsCatalogue::getArchiveFilesItor(const TapeFileSea
try {
// Create a connection to populate the temporary table (specialised by database type)
auto conn = m_archiveFileListingConnPool.getConn();
- const auto tempDiskFxidsTableName = createAndPopulateTempTableFxid(conn, searchCriteria);
+ const auto tempDiskFxidsTableName = createAndPopulateTempTableFxid(conn, searchCriteria.diskFileIds);
// Pass ownership of the connection to the Iterator object
auto impl = new RdbmsCatalogueGetArchiveFilesItor(m_log, std::move(conn), searchCriteria, tempDiskFxidsTableName);
return ArchiveFileItor(impl);
@@ -7005,9 +7006,8 @@ Catalogue::ArchiveFileItor RdbmsCatalogue::getTapeContentsItor(const std::string
//------------------------------------------------------------------------------
// checkRecycleTapeFileSearchCriteria
//------------------------------------------------------------------------------
-void RdbmsCatalogue::checkRecycleTapeFileSearchCriteria(const RecycleTapeFileSearchCriteria & searchCriteria) const {
+void RdbmsCatalogue::checkRecycleTapeFileSearchCriteria(cta::rdbms::Conn &conn, const RecycleTapeFileSearchCriteria & searchCriteria) const {
if(searchCriteria.vid) {
- auto conn = m_connPool.getConn();
if(!tapeExists(conn, searchCriteria.vid.value())) {
throw exception::UserError(std::string("Tape ") + searchCriteria.vid.value() + " does not exist");
}
@@ -7016,8 +7016,10 @@ void RdbmsCatalogue::checkRecycleTapeFileSearchCriteria(const RecycleTapeFileSea
Catalogue::FileRecycleLogItor RdbmsCatalogue::getFileRecycleLogItor(const RecycleTapeFileSearchCriteria & searchCriteria) const {
try {
- checkRecycleTapeFileSearchCriteria(searchCriteria);
- auto impl = new RdbmsCatalogueGetFileRecycleLogItor(m_log, m_archiveFileListingConnPool, searchCriteria);
+ auto conn = m_archiveFileListingConnPool.getConn();
+ checkRecycleTapeFileSearchCriteria(conn, searchCriteria);
+ const auto tempDiskFxidsTableName = createAndPopulateTempTableFxid(conn, searchCriteria.diskFileIds);
+ auto impl = new RdbmsCatalogueGetFileRecycleLogItor(m_log, std::move(conn), searchCriteria, tempDiskFxidsTableName);
return FileRecycleLogItor(impl);
} catch(exception::UserError &) {
throw;
@@ -7185,7 +7187,7 @@ common::dataStructures::ArchiveFileSummary RdbmsCatalogue::getTapeFileSummary(
addedAWhereConstraint = true;
}
if(searchCriteria.diskFileIds) {
- const auto tempDiskFxidsTableName = createAndPopulateTempTableFxid(conn, searchCriteria);
+ const auto tempDiskFxidsTableName = createAndPopulateTempTableFxid(conn, searchCriteria.diskFileIds);
if(addedAWhereConstraint) sql += " AND ";
sql += "ARCHIVE_FILE.DISK_FILE_ID IN (SELECT DISK_FILE_ID FROM " + tempDiskFxidsTableName + ")";
@@ -7220,6 +7222,80 @@ common::dataStructures::ArchiveFileSummary RdbmsCatalogue::getTapeFileSummary(
}
}
+
+//------------------------------------------------------------------------------
+// deleteTapeFileCopy
+//------------------------------------------------------------------------------
+void RdbmsCatalogue::deleteTapeFileCopy(const std::string &vid, const uint64_t archiveFileId, const std::string &reason) {
+ try {
+ TapeFileSearchCriteria searchCriteria;
+ searchCriteria.archiveFileId = archiveFileId;
+ deleteTapeFileCopy(vid, searchCriteria, reason);
+ } catch(exception::UserError &) {
+ throw;
+ } catch(exception::Exception &ex) {
+ ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str());
+ throw;
+ }
+}
+
+
+//------------------------------------------------------------------------------
+// deleteTapeFileCopy
+//------------------------------------------------------------------------------
+void RdbmsCatalogue::deleteTapeFileCopy(const std::string &vid, const std::string &diskFileId,
+ const std::string &diskInstanceName, const std::string &reason) {
+ try {
+ // cta-admin converts the list from EOS fxid (hex) to fid (dec). In the case of the
+ // single option on the command line we need to do the conversion ourselves.
+ auto fid = strtol(diskFileId.c_str(), nullptr, 16);
+ if(fid < 1 || fid == LONG_MAX) {
+ throw cta::exception::UserError(diskFileId + " is not a valid file ID");
+ }
+ TapeFileSearchCriteria searchCriteria;
+ searchCriteria.diskInstance = diskInstanceName;
+
+ searchCriteria.diskFileIds = std::vector<std::string>();
+ searchCriteria.diskFileIds->push_back(std::to_string(fid));
+ deleteTapeFileCopy(vid, searchCriteria, reason);
+ } catch(exception::UserError &) {
+ throw;
+ } catch(exception::Exception &ex) {
+ ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str());
+ throw;
+ }
+}
+
+
+//------------------------------------------------------------------------------
+// deleteTapeFileCopy
+//------------------------------------------------------------------------------
+void RdbmsCatalogue::deleteTapeFileCopy(const std::string &vid, const TapeFileSearchCriteria &criteria,
+ const std::string &reason) {
+ auto itor = getArchiveFilesItor(criteria); // itor should have only at most one archive file since we always search on unique attributes
+ if (!itor.hasMore()) {
+ throw exception::UserError(std::string("Cannot delete the copy because the file specified does not exist"));
+ }
+ cta::common::dataStructures::ArchiveFile af = itor.next();
+ if (af.tapeFiles.size() == 1) {
+ throw exception::UserError(std::string("Cannot delete the copy because it is the only copy"));
+ }
+
+ af.tapeFiles.removeAllVidsExcept(vid); // assume there is only one copy per vid, this should return a list with at most one item
+ if (af.tapeFiles.empty()) {
+ throw exception::UserError(std::string("No copy present on the specified vid"));
+ }
+ if (af.tapeFiles.size() > 1){
+ throw exception::UserError(std::string("Error: More than one copy of the file in the specified vid"));
+ }
+
+ log::LogContext lc(m_log);
+ auto conn = m_connPool.getConn();
+ af.diskFileInfo.path = "Not applicable for copies deleted with cta-admin tf rm"; // will go into the diskFilePath column of the File Recycle log
+ copyTapeFileToFileRecyleLogAndDelete(conn, af, reason, lc);
+}
+
+
//------------------------------------------------------------------------------
// getArchiveFileById
//------------------------------------------------------------------------------
@@ -8818,6 +8894,32 @@ void RdbmsCatalogue::copyArchiveFileToFileRecycleLog(rdbms::Conn & conn, const c
}
}
+
+//------------------------------------------------------------------------------
+// copyTapeFilesToFileRecycleLog
+//------------------------------------------------------------------------------
+void RdbmsCatalogue::copyTapeFilesToFileRecycleLog(rdbms::Conn & conn, const common::dataStructures::ArchiveFile &archiveFile, const std::string &reason) {
+ try {
+ for(auto &tapeFile: archiveFile.tapeFiles) {
+ //Create one file recycle log entry per tape file
+ InsertFileRecycleLog fileRecycleLog;
+ fileRecycleLog.vid = tapeFile.vid;
+ fileRecycleLog.fSeq = tapeFile.fSeq;
+ fileRecycleLog.blockId = tapeFile.blockId;
+ fileRecycleLog.copyNb = tapeFile.copyNb;
+ fileRecycleLog.tapeFileCreationTime = tapeFile.creationTime;
+ fileRecycleLog.archiveFileId = archiveFile.archiveFileID;
+ fileRecycleLog.diskFilePath = archiveFile.diskFileInfo.path;
+ fileRecycleLog.reasonLog = "(Deleted using cta-admin tf rm) " + reason;
+ fileRecycleLog.recycleLogTime = time(nullptr);
+ insertFileInFileRecycleLog(conn,fileRecycleLog);
+ }
+ } catch(exception::Exception &ex) {
+ ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str());
+ throw;
+ }
+}
+
//------------------------------------------------------------------------------
// insertFileInFileRecycleLog
//------------------------------------------------------------------------------
@@ -8913,6 +9015,32 @@ void RdbmsCatalogue::deleteTapeFiles(rdbms::Conn & conn, const common::dataStruc
}
}
+void RdbmsCatalogue::deleteTapeFiles(rdbms::Conn & conn, const common::dataStructures::ArchiveFile& file){
+ try {
+ for(auto &tapeFile: file.tapeFiles) {
+
+ //Delete the tape file.
+ const char *const deleteTapeFilesSql =
+ "DELETE FROM "
+ "TAPE_FILE "
+ "WHERE "
+ "TAPE_FILE.VID = :VID AND "
+ "TAPE_FILE.FSEQ = :FSEQ";
+
+ auto deleteTapeFilesStmt = conn.createStmt(deleteTapeFilesSql);
+ deleteTapeFilesStmt.bindString(":VID", tapeFile.vid);
+ deleteTapeFilesStmt.bindUint64(":FSEQ", tapeFile.fSeq);
+ deleteTapeFilesStmt.executeNonQuery();
+ }
+ } catch(exception::UserError &) {
+ throw;
+ } catch(exception::Exception &ex) {
+ ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str());
+ throw;
+ }
+}
+
+
void RdbmsCatalogue::deleteArchiveFile(rdbms::Conn& conn, const common::dataStructures::DeleteArchiveRequest& request){
try{
const char *const deleteArchiveFileSql =
diff --git a/catalogue/RdbmsCatalogue.hpp b/catalogue/RdbmsCatalogue.hpp
index bc49ae9dd0736f36c3eed96c797dd620e29ad3e6..ecb4934b741285eba88d38f4d8ff9d59f8fa5660 100644
--- a/catalogue/RdbmsCatalogue.hpp
+++ b/catalogue/RdbmsCatalogue.hpp
@@ -851,10 +851,10 @@ public:
/**
* Throws a UserError exception if the specified searchCriteria is not valid
* due to a user error.
- *
+ * @param conn The database connection.
* @param searchCriteria The search criteria.
*/
- void checkRecycleTapeFileSearchCriteria(const RecycleTapeFileSearchCriteria & searchCriteria) const;
+ void checkRecycleTapeFileSearchCriteria(cta::rdbms::Conn &conn, const RecycleTapeFileSearchCriteria & searchCriteria) const;
/**
* Returns all the currently deleted files by looking at the FILE_RECYCLE_LOG table
@@ -902,7 +902,39 @@ public:
common::dataStructures::ArchiveFileSummary getTapeFileSummary(
const TapeFileSearchCriteria &searchCriteria) const override;
+ /**
+ * Deletes a tape file copy
+ *
+ * @param vid The vid of the tape the file copy is stored on
+ * @param archiveFileId The unique identifier of the archived file
+ * @param reason The reason for deleting the tape file copy
+ */
+ void deleteTapeFileCopy(const std::string &vid, const uint64_t archiveFileId, const std::string &reason) override;
+
+ /**
+ * Deletes a tape file copy
+ *
+ * @param vid The vid of the tape the file copy is stored on
+ * @param diskFileId The identifier of the archive_file on disk
+ * @param diskInstanceName The name of the archived file disk instance
+ * @param reason The reason for deleting the tape file copy
+ */
+ void deleteTapeFileCopy(const std::string &vid, const std::string &diskFileId,
+ const std::string &diskInstanceName, const std::string &reason) override;
+
+
/**
+ * Deletes a tape file copy
+ *
+ * @param criteria The search criteria of the archive file
+ * @param vid The vid of the tape the file copy is stored on
+ * @param criteria The search criteria of the archive file
+ * @param reason The reason for deleting the tape file copy
+ */
+ void deleteTapeFileCopy(const std::string &vid, const TapeFileSearchCriteria &criteria, const std::string &reason);
+
+
+ /**
* Returns the archive file with the specified unique identifier.
*
* This method assumes that the archive file being requested exists and will
@@ -1575,10 +1607,10 @@ protected:
* Creates a temporary table from the list of disk file IDs provided in the search criteria.
*
* @param conn The database connection.
- * @param tapeFileSearchCriteria Search criteria containing a list of disk file IDs (fxid).
+ * @param diskFileIds List of disk file IDs (fxid).
* @return Name of the temporary table
*/
- virtual std::string createAndPopulateTempTableFxid(rdbms::Conn &conn, const TapeFileSearchCriteria &tapeFileSearchCriteria) const = 0;
+ virtual std::string createAndPopulateTempTableFxid(rdbms::Conn &conn, const optional<std::vector<std::string>> &diskFileIds) const = 0;
/**
* Returns a unique archive ID that can be used by a new archive file within
@@ -1898,6 +1930,24 @@ protected:
*/
void copyArchiveFileToFileRecycleLog(rdbms::Conn & conn, const common::dataStructures::DeleteArchiveRequest & request);
+ /**
+ * Copies the TAPE_FILE entries to the recycle-bin tables
+ * @param conn the database connection
+ * @param file the archiveFile whose tapefiles we want to copy
+ * @param reason The reason for deleting the tape file copy
+ */
+ void copyTapeFilesToFileRecycleLog(rdbms::Conn & conn, const common::dataStructures::ArchiveFile &file, const std::string &reason);
+
+ /**
+ * Copy the tape files from the TAPE_FILE tables to the FILE_RECYCLE_LOG table
+ * and deletes the TAPE_FILE entry.
+ * @param conn the database connection
+ * @param file the archive file containing the tapefile to be copied
+ * @param reason The reason for deleting the tape file copy
+ * @param lc the log context
+ */
+ virtual void copyTapeFileToFileRecyleLogAndDelete(rdbms::Conn & conn, const cta::common::dataStructures::ArchiveFile &file, const std::string &reason, log::LogContext & lc) = 0;
+
/**
* Insert the file in the FILE_RECYCLE_LOG table
* @param conn the database connection
@@ -1919,6 +1969,13 @@ protected:
*/
void deleteTapeFiles(rdbms::Conn & conn, const common::dataStructures::DeleteArchiveRequest & request);
+ /**
+ * Delete the TapeFiles associated to an ArchiveFile from the TAPE_FILE table
+ * @param conn the database connection
+ * @param file the file that contains the tape files to delete
+ */
+ void deleteTapeFiles(rdbms::Conn & conn, const common::dataStructures::ArchiveFile &file);
+
/**
* Set the DIRTY flag to true
* @param conn the database connection
diff --git a/catalogue/RdbmsCatalogueGetFileRecycleLogItor.cpp b/catalogue/RdbmsCatalogueGetFileRecycleLogItor.cpp
index ab9b81519cea0155e55f7bf0bcd6bd876141e2eb..71f66cb27fb918d9b66512e0eb22cffddc4c8a85 100644
--- a/catalogue/RdbmsCatalogueGetFileRecycleLogItor.cpp
+++ b/catalogue/RdbmsCatalogueGetFileRecycleLogItor.cpp
@@ -27,10 +27,11 @@ namespace catalogue {
//------------------------------------------------------------------------------
RdbmsCatalogueGetFileRecycleLogItor::RdbmsCatalogueGetFileRecycleLogItor(
log::Logger &log,
- rdbms::ConnPool &connPool,
- const RecycleTapeFileSearchCriteria & searchCriteria):
+ rdbms::Conn &&conn,
+ const RecycleTapeFileSearchCriteria & searchCriteria,
+ const std::string &tempDiskFxidsTableName):
m_log(log),
- m_connPool(connPool),
+ m_conn(std::move(conn)),
m_searchCriteria(searchCriteria),
m_rsetIsEmpty(true),
m_hasMoreHasBeenCalled(false) {
@@ -65,7 +66,10 @@ RdbmsCatalogueGetFileRecycleLogItor::RdbmsCatalogueGetFileRecycleLogItor(
const bool thereIsAtLeastOneSearchCriteria =
searchCriteria.vid ||
- searchCriteria.diskFileId;
+ searchCriteria.diskFileIds ||
+ searchCriteria.archiveFileId ||
+ searchCriteria.copynb ||
+ searchCriteria.diskInstance;
if(thereIsAtLeastOneSearchCriteria) {
sql += " WHERE ";
@@ -77,12 +81,29 @@ RdbmsCatalogueGetFileRecycleLogItor::RdbmsCatalogueGetFileRecycleLogItor(
sql += "FILE_RECYCLE_LOG.VID = :VID";
addedAWhereConstraint = true;
}
+
+ if (searchCriteria.archiveFileId) {
+ if(addedAWhereConstraint) sql += " AND ";
+ sql += "FILE_RECYCLE_LOG.ARCHIVE_FILE_ID = :ARCHIVE_FILE_ID";
+ addedAWhereConstraint = true;
+ }
- if(searchCriteria.diskFileId){
+ if(searchCriteria.diskFileIds) {
if(addedAWhereConstraint) sql += " AND ";
- sql += "FILE_RECYCLE_LOG.DISK_FILE_ID = :DISK_FILE_ID";
+ sql += "FILE_RECYCLE_LOG.DISK_FILE_ID IN (SELECT DISK_FILE_ID FROM " + tempDiskFxidsTableName + ")";
addedAWhereConstraint = true;
}
+
+ if (searchCriteria.diskInstance) {
+ if(addedAWhereConstraint) sql += " AND ";
+ sql += "FILE_RECYCLE_LOG.DISK_INSTANCE_NAME = :DISK_INSTANCE";
+ addedAWhereConstraint = true;
+ }
+
+ if (searchCriteria.copynb) {
+ if(addedAWhereConstraint) sql += " AND ";
+ sql += "FILE_RECYCLE_LOG.COPY_NB = :COPY_NB";
+ }
// Order by FSEQ if we are listing the contents of a tape, else order by archive file ID
if(searchCriteria.vid) {
@@ -91,15 +112,22 @@ RdbmsCatalogueGetFileRecycleLogItor::RdbmsCatalogueGetFileRecycleLogItor(
sql += " ORDER BY FILE_RECYCLE_LOG.ARCHIVE_FILE_ID, FILE_RECYCLE_LOG.COPY_NB";
}
- m_conn = connPool.getConn();
m_stmt = m_conn.createStmt(sql);
if(searchCriteria.vid){
m_stmt.bindString(":VID", searchCriteria.vid.value());
}
-
- if(searchCriteria.diskFileId){
- m_stmt.bindString(":DISK_FILE_ID", searchCriteria.diskFileId.value());
+
+ if (searchCriteria.archiveFileId) {
+ m_stmt.bindUint64(":ARCHIVE_FILE_ID", searchCriteria.archiveFileId.value());
+ }
+
+ if (searchCriteria.diskInstance) {
+ m_stmt.bindString(":DISK_INSTANCE", searchCriteria.diskInstance.value());
+ }
+
+ if (searchCriteria.copynb) {
+ m_stmt.bindUint64(":COPY_NB", searchCriteria.copynb.value());
}
m_rset = m_stmt.executeQuery();
diff --git a/catalogue/RdbmsCatalogueGetFileRecycleLogItor.hpp b/catalogue/RdbmsCatalogueGetFileRecycleLogItor.hpp
index c94cfe3949fa916190a23e0c55ca9ff4e01f993a..90e996b82ed39506e25cc28634f225bd7106d5b6 100644
--- a/catalogue/RdbmsCatalogueGetFileRecycleLogItor.hpp
+++ b/catalogue/RdbmsCatalogueGetFileRecycleLogItor.hpp
@@ -29,9 +29,13 @@ public:
* Constructor.
*
* @param log Object representing the API to the CTA logging system.
- * @param connPool The database connection pool.
+ * @param conn A database connection.
*/
- RdbmsCatalogueGetFileRecycleLogItor(log::Logger &log, rdbms::ConnPool &connPool, const RecycleTapeFileSearchCriteria & searchCriteria);
+ RdbmsCatalogueGetFileRecycleLogItor(
+ log::Logger &log,
+ rdbms::Conn &&conn,
+ const RecycleTapeFileSearchCriteria & searchCriteria,
+ const std::string &tempDiskFxidsTableName);
/**
* Destructor.
@@ -54,10 +58,12 @@ private:
*/
log::Logger &m_log;
+
/**
- * The database connection pool.
+ * The database connection.
*/
- rdbms::ConnPool &m_connPool;
+ rdbms::Conn m_conn;
+
/**
* The search criteria to be used when listing recycled tape files.
@@ -79,11 +85,6 @@ private:
*/
bool m_hasMoreHasBeenCalled;
- /**
- * The database connection.
- */
- rdbms::Conn m_conn;
-
/**
* The database statement.
*/
diff --git a/catalogue/RecyleTapeFileSearchCriteria.hpp b/catalogue/RecyleTapeFileSearchCriteria.hpp
index 8fc49830645e51f08c5c95006a235c8491ff22a3..eb4ab60cb73af6c73b9d52a7895e0082adad13ac 100644
--- a/catalogue/RecyleTapeFileSearchCriteria.hpp
+++ b/catalogue/RecyleTapeFileSearchCriteria.hpp
@@ -32,6 +32,15 @@ namespace catalogue {
* Please note that no wild cards, for example '*' or '%', are supported.
*/
struct RecycleTapeFileSearchCriteria {
+ /**
+ * The unique identifier of an archive file.
+ */
+ optional<uint64_t> archiveFileId;
+
+ /**
+ * The name of a disk instance.
+ */
+ optional<std::string> diskInstance;
/**
* The volume identifier of a tape.
@@ -44,7 +53,12 @@ struct RecycleTapeFileSearchCriteria {
* These are given as a list of strings in DECIMAL format. EOS provides the fxids in hex format. The parsing and
* conversion into decimal is done in the cta-admin client, ready to be built into a SQL query string.
*/
- optional<std::string> diskFileId;
+ optional<std::vector<std::string>> diskFileIds;
+
+ /**
+ * The copy number of the deleted tape file.
+ */
+ optional<uint64_t> copynb;
}; // struct TapeFileSearchCriteria
diff --git a/catalogue/SqliteCatalogue.cpp b/catalogue/SqliteCatalogue.cpp
index c809025876f84e27a19ee67f5141488b3046009c..f235c0279b3e224e75d9b0ea14512e7373091b3b 100644
--- a/catalogue/SqliteCatalogue.cpp
+++ b/catalogue/SqliteCatalogue.cpp
@@ -192,7 +192,7 @@ void SqliteCatalogue::DO_NOT_USE_deleteArchiveFile_DO_NOT_USE(const std::string
//------------------------------------------------------------------------------
// createAndPopulateTempTableFxid
//------------------------------------------------------------------------------
-std::string SqliteCatalogue::createAndPopulateTempTableFxid(rdbms::Conn &conn, const TapeFileSearchCriteria &tapeFileSearchCriteria) const {
+std::string SqliteCatalogue::createAndPopulateTempTableFxid(rdbms::Conn &conn, const optional<std::vector<std::string>> &diskFileIds) const {
try {
const std::string tempTableName = "TEMP.DISK_FXIDS";
@@ -200,9 +200,9 @@ std::string SqliteCatalogue::createAndPopulateTempTableFxid(rdbms::Conn &conn, c
conn.executeNonQuery("DROP TABLE IF EXISTS " + tempTableName);
conn.executeNonQuery("CREATE TEMPORARY TABLE " + tempTableName + "(DISK_FILE_ID TEXT)");
- if(tapeFileSearchCriteria.diskFileIds) {
+ if(diskFileIds) {
auto stmt = conn.createStmt("INSERT INTO " + tempTableName + " VALUES(:DISK_FILE_ID)");
- for(auto &diskFileId : tapeFileSearchCriteria.diskFileIds.value()) {
+ for(auto &diskFileId : diskFileIds.value()) {
stmt.bindString(":DISK_FILE_ID", diskFileId);
stmt.executeNonQuery();
}
@@ -650,6 +650,40 @@ void SqliteCatalogue::deleteTapeFilesAndArchiveFileFromRecycleBin(rdbms::Conn& c
}
}
+//------------------------------------------------------------------------------
+// copyTapeFileToFileRecyleLogAndDelete
+//------------------------------------------------------------------------------
+void SqliteCatalogue::copyTapeFileToFileRecyleLogAndDelete(rdbms::Conn & conn, const cta::common::dataStructures::ArchiveFile &file,
+ const std::string &reason, log::LogContext & lc) {
+ try {
+ utils::Timer t;
+ log::TimingList tl;
+ //We currently do an INSERT INTO and a DELETE FROM
+ //in a single transaction
+ conn.executeNonQuery("BEGIN TRANSACTION");
+ copyTapeFilesToFileRecycleLog(conn, file, reason);
+ tl.insertAndReset("insertToRecycleBinTime",t);
+ setTapeDirty(conn, file.archiveFileID);
+ tl.insertAndReset("setTapeDirtyTime",t);
+ deleteTapeFiles(conn,file);
+ tl.insertAndReset("deleteTapeFilesTime",t);
+ conn.commit();
+ tl.insertAndReset("commitTime",t);
+ log::ScopedParamContainer spc(lc);
+ spc.add("archiveFileId", file.archiveFileID);
+ spc.add("diskFileId", file.diskFileId);
+ spc.add("diskFilePath", file.diskFileInfo.path);
+ spc.add("diskInstance", file.diskInstance);
+ tl.addToLog(spc);
+ lc.log(log::INFO,"In SqliteCatalogue::copyArchiveFileToRecycleBinAndDelete: ArchiveFile moved to the recycle-bin.");
+
+ } catch(exception::UserError &) {
+ throw;
+ } catch(exception::Exception &ex) {
+ ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str());
+ throw;
+ }
+}
} // namespace catalogue
} // namespace cta
diff --git a/catalogue/SqliteCatalogue.hpp b/catalogue/SqliteCatalogue.hpp
index 020589f930823f08d39bd77e288d0a703a3a11a6..4102a8532ba2b638de1643b88859246f99aed3aa 100644
--- a/catalogue/SqliteCatalogue.hpp
+++ b/catalogue/SqliteCatalogue.hpp
@@ -86,10 +86,10 @@ protected:
* Creates a temporary table from the list of disk file IDs provided in the search criteria.
*
* @param conn The database connection.
- * @param tapeFileSearchCriteria Search criteria containing a list of disk file IDs (fxid).
+ * @param diskFileIds List of disk file IDs (fxid).
* @return Name of the temporary table
*/
- std::string createAndPopulateTempTableFxid(rdbms::Conn &conn, const TapeFileSearchCriteria &tapeFileSearchCriteria) const override;
+ std::string createAndPopulateTempTableFxid(rdbms::Conn &conn, const optional<std::vector<std::string>> &diskFileIds) const override;
/**
* Returns a unique archive ID that can be used by a new archive file within
@@ -216,6 +216,17 @@ protected:
*/
void deleteTapeFilesAndArchiveFileFromRecycleBin(rdbms::Conn& conn, const uint64_t archiveFileId, log::LogContext& lc) override;
+ /**
+ * Copy the tape files from the TAPE_FILE tables to the FILE_RECYCLE_LOG table
+ * and deletes the TAPE_FILE entry.
+ * @param conn the database connection
+ * @param file the file to be deleted
+ * @param reason The reason for deleting the tape file copy
+ * @param lc the log context
+ */
+ void copyTapeFileToFileRecyleLogAndDelete(rdbms::Conn & conn, const cta::common::dataStructures::ArchiveFile &file,
+ const std::string &reason, log::LogContext & lc) override;
+
private:
/**
diff --git a/cmdline/CtaAdminCmdParse.hpp b/cmdline/CtaAdminCmdParse.hpp
index 70b6123de3d2a67a453fdecac8e0321c38022608..2231f863910a52ac1087d56e44e0125c1447b9b6 100644
--- a/cmdline/CtaAdminCmdParse.hpp
+++ b/cmdline/CtaAdminCmdParse.hpp
@@ -371,10 +371,12 @@ const std::map<AdminCmd::Cmd, CmdHelp> cmdHelp = {
{ AdminCmd::CMD_SHOWQUEUES, { "showqueues", "sq", { } }},
{ AdminCmd::CMD_STORAGECLASS, { "storageclass", "sc", { "add", "ch", "rm", "ls" } }},
{ AdminCmd::CMD_TAPE, { "tape", "ta", { "add", "ch", "rm", "reclaim", "ls", "label" } }},
- { AdminCmd::CMD_TAPEFILE, { "tapefile", "tf", { "ls" },
+ { AdminCmd::CMD_TAPEFILE, { "tapefile", "tf", { "ls", "rm" },
+ "\n This command allows to manage files stored on tape\n"
" Tape files can be listed by VID or by EOS disk instance + EOS disk file ID.\n"
" Disk file IDs should be provided in hexadecimal (fxid). The --fxidfile option\n"
- " takes a file in the same format as the output of 'eos find --fid <path>'\n\n"
+ " takes a file in the same format as the output of 'eos find --fid <path>'\n"
+ " Delete a file copy with the \"rm\" subcommand\n\n"
}},
{ AdminCmd::CMD_TAPEPOOL, { "tapepool", "tp", { "add", "ch", "rm", "ls" } }},
{ AdminCmd::CMD_DISKSYSTEM, { "disksystem", "ds", { "add", "ch", "rm", "ls" },
@@ -408,7 +410,7 @@ const std::map<AdminCmd::Cmd, CmdHelp> cmdHelp = {
{ AdminCmd::CMD_VERSION, { "version", "v", { } }},
{ AdminCmd::CMD_SCHEDULINGINFOS, { "schedulinginfo", "si", { "ls" } }},
{ AdminCmd::CMD_RECYCLETAPEFILE, { "recycletf", "rtf", { "ls" },
- " Tape files in the recycle log can be listed by VID or by EOS disk file ID.\n"
+ " Tape files in the recycle log can be listed by VID, EOS disk file ID, EOS disk instance, ArchiveFileId or copy number.\n"
" Disk file IDs should be provided in hexadecimal (fxid).\n\n" }},
};
@@ -587,6 +589,8 @@ const std::map<cmd_key_t, cmd_val_t> cmdOptions = {
{{ AdminCmd::CMD_TAPEFILE, AdminCmd::SUBCMD_LS },
{ opt_vid.optional(), opt_instance.optional(), opt_fid.optional(), opt_fidfile.optional(),
opt_lookupns.optional(), opt_archivefileid.optional() }},
+ {{ AdminCmd::CMD_TAPEFILE, AdminCmd::SUBCMD_RM },
+ { opt_vid, opt_instance.optional(), opt_fid.optional(), opt_archivefileid.optional(), opt_reason }},
/*----------------------------------------------------------------------------------------------------*/
{{ AdminCmd::CMD_TAPEPOOL, AdminCmd::SUBCMD_ADD },
{ opt_tapepool_alias, opt_vo, opt_partialtapes, opt_encrypted, opt_supply.optional(), opt_comment }},
@@ -614,7 +618,8 @@ const std::map<cmd_key_t, cmd_val_t> cmdOptions = {
{ }},
{{ AdminCmd::CMD_VERSION, AdminCmd::SUBCMD_NONE }, { }},
{{ AdminCmd::CMD_SCHEDULINGINFOS, AdminCmd::SUBCMD_LS }, { }},
- {{ AdminCmd::CMD_RECYCLETAPEFILE, AdminCmd::SUBCMD_LS }, { opt_vid.optional(), opt_fid.optional() }},
+ {{ AdminCmd::CMD_RECYCLETAPEFILE, AdminCmd::SUBCMD_LS },
+ { opt_vid.optional(), opt_fid.optional(), opt_fidfile.optional(), opt_copynb.optional(), opt_archivefileid.optional(), opt_instance.optional() }},
};
diff --git a/cmdline/CtaAdminTextFormatter.cpp b/cmdline/CtaAdminTextFormatter.cpp
index 4083520fb6825c50b57d29ba6dbce118cbaa4fbf..993e96bb14dcb6572b1146aafcf9db0c08ac595b 100644
--- a/cmdline/CtaAdminTextFormatter.cpp
+++ b/cmdline/CtaAdminTextFormatter.cpp
@@ -121,17 +121,16 @@ void TextFormatter::flush() {
}
// Output columns
- bool lastColumnFlushLeft = false;
for(auto &l : m_outputBuffer) {
if(is_header) { std::cout << TEXT_RED; }
for(size_t c = 0; c < l.size(); ++c) {
// flush right, except for comments, paths and drive reasons, which are flush left
if(is_header && c == l.size()-1 &&
(l.at(c) == "comment" || l.at(c) == "path" || l.at(c) == "reason")) {
- lastColumnFlushLeft = true;
+ m_lastColumnFlushLeft = true;
}
- auto flush = (c == l.size()-1 && lastColumnFlushLeft) ? std::left : std::right;
+ auto flush = (c == l.size()-1 && m_lastColumnFlushLeft) ? std::left : std::right;
std::cout << std::setfill(' ')
<< std::setw(m_colSize.at(c))
diff --git a/cmdline/CtaAdminTextFormatter.hpp b/cmdline/CtaAdminTextFormatter.hpp
index e3745d454cdf0cc7f3c2207609868b36d21f3f7d..26f0a86f707804e1c8ef3c6ee003b5e3d468b2b8 100644
--- a/cmdline/CtaAdminTextFormatter.hpp
+++ b/cmdline/CtaAdminTextFormatter.hpp
@@ -35,6 +35,7 @@ public:
TextFormatter(unsigned int bufLines = 1000) :
m_bufLines(bufLines) {
m_outputBuffer.reserve(bufLines);
+ m_lastColumnFlushLeft = false;
}
~TextFormatter() {
@@ -157,7 +158,7 @@ private:
std::vector<unsigned int> m_colSize; //!< Array of column sizes
unsigned int m_bufLines; //!< Number of text lines to buffer before flushing formatted output
std::vector<std::vector<std::string>> m_outputBuffer; //!< Buffer for text output (not used for JSON)
-
+ bool m_lastColumnFlushLeft; //!< Flag indicating if last collumn should be aligned left
static constexpr const char* const TEXT_RED = "\x1b[31;1m"; //!< Terminal formatting code for red text
static constexpr const char* const TEXT_NORMAL = "\x1b[0m"; //!< Terminal formatting code for normal text
static constexpr const int NB_CHAR_REASON = 50; //!< Reason max length to display in tabular output (DriveLs and TapeLs)
diff --git a/cta-release/CMakeLists.txt b/cta-release/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..965999a806691a3a9542b0cddd7f92fa21d857b9
--- /dev/null
+++ b/cta-release/CMakeLists.txt
@@ -0,0 +1,76 @@
+# @project The CERN Tape Archive (CTA)
+# @copyright Copyright(C) 2015-2021 CERN
+# @license This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+cmake_minimum_required (VERSION 2.6)
+
+# As file(DOWNLOAD) fails silently
+function(safedl SOURCEURL DESTFILE)
+ file(DOWNLOAD "${SOURCEURL}"
+ "${DESTFILE}"
+ STATUS status
+ )
+
+ list(GET status 0 status_code)
+ list(GET status 1 status_string)
+
+ if(NOT status_code EQUAL 0)
+ message(WARNING "error:
+ downloading ${SOURCEURL} failed
+ ${status_string}
+ ")
+ endif()
+endfunction()
+
+# We need the el version to select the proper key for Oracle repo
+include(../cmake/UseRPMToolsEnvironment.cmake)
+
+if( "${RPMTools_RPMBUILD_DIST}" MATCHES "\\.el([0-9])\\.")
+ set(OSV "${CMAKE_MATCH_1}")
+else()
+# Default to el7
+ set(OSV "7")
+endif()
+
+# Download package signing keys
+safedl("https://storage-ci.web.cern.ch/storage-ci/storageci.key"
+ "${CMAKE_CURRENT_SOURCE_DIR}/RPM-GPG-KEY-storageci"
+)
+
+safedl("https://download.ceph.com/keys/release.asc"
+ "${CMAKE_CURRENT_SOURCE_DIR}/RPM-GPG-KEY-ceph"
+)
+
+safedl("https://yum.oracle.com/RPM-GPG-KEY-oracle-ol${OSV}"
+ "${CMAKE_CURRENT_SOURCE_DIR}/RPM-GPG-KEY-oracle"
+)
+
+# Repos files
+file (GLOB REPO_FILES
+ "${CMAKE_CURRENT_SOURCE_DIR}/*.repo"
+)
+
+# Signing keys
+file (GLOB KEY_FILES
+ "${CMAKE_CURRENT_SOURCE_DIR}/RPM-GPG-KEY-*"
+)
+
+# Install package files
+install (FILES ${REPO_FILES}
+ DESTINATION ${CMAKE_INSTALL_SYSCONFDIR}/yum.repos.d)
+install (FILES ${KEY_FILES}
+ DESTINATION ${CMAKE_INSTALL_SYSCONFDIR}/pki/rpm-gpg)
+install (FILES ../continuousintegration/docker/ctafrontend/cc7/etc/yum/pluginconf.d/versionlock.list
+ DESTINATION ${CMAKE_INSTALL_SYSCONFDIR}/yum/pluginconf.d
+ RENAME versionlock.cta)
+
diff --git a/cta-release/ceph.repo b/cta-release/ceph.repo
new file mode 100644
index 0000000000000000000000000000000000000000..8dcb73669669416297f2e50304c4fa671583d86e
--- /dev/null
+++ b/cta-release/ceph.repo
@@ -0,0 +1,8 @@
+[Ceph]
+name=Ceph packages for $basearch
+baseurl=http://download.ceph.com/rpm-nautilus/el$releasever/$basearch
+enabled=1
+gpgcheck=1
+protect=1
+type=rpm-md
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-ceph
diff --git a/cta-release/cta.repo b/cta-release/cta.repo
new file mode 100644
index 0000000000000000000000000000000000000000..54a744cd6c33649ff1d98356f0436e9de9bdacd2
--- /dev/null
+++ b/cta-release/cta.repo
@@ -0,0 +1,6 @@
+[cta]
+name=CTA releases from CTA project
+baseurl=https://cta-repo.web.cern.ch/cta-repo/
+enabled=1
+gpgcheck=0
+priority=10
diff --git a/cta-release/eos-citrine-depend.repo b/cta-release/eos-citrine-depend.repo
new file mode 100644
index 0000000000000000000000000000000000000000..cd56da935595daf3e65a8841a03855635e1aefd3
--- /dev/null
+++ b/cta-release/eos-citrine-depend.repo
@@ -0,0 +1,6 @@
+[eos-citrine-depend]
+name=dependencies for EOS citrine releases from EOS project
+baseurl=http://storage-ci.web.cern.ch/storage-ci/eos/citrine-depend/el-$releasever/$basearch/
+enabled=1
+gpgcheck=0
+priority=10
diff --git a/cta-release/eos-citrine.repo b/cta-release/eos-citrine.repo
new file mode 100644
index 0000000000000000000000000000000000000000..67b7de8d00a209345f0725eeacca2af06a86bb86
--- /dev/null
+++ b/cta-release/eos-citrine.repo
@@ -0,0 +1,7 @@
+[eos-citrine]
+name=EOS citrine releases from EOS project
+baseurl=http://storage-ci.web.cern.ch/storage-ci/eos/citrine/tag/testing/el-$releasever/$basearch/
+enabled=1
+gpgcheck=1
+priority=10
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-storageci
diff --git a/cta-release/eos-quarkdb.repo b/cta-release/eos-quarkdb.repo
new file mode 100644
index 0000000000000000000000000000000000000000..9137a0932924e4158c050f3608a7999e954f1ed7
--- /dev/null
+++ b/cta-release/eos-quarkdb.repo
@@ -0,0 +1,7 @@
+[eos-quarkdb]
+name=EOS quarkdb releases from EOS project
+baseurl=http://storage-ci.web.cern.ch/storage-ci/quarkdb/tag/el$releasever/$basearch/
+enabled=1
+gpgcheck=1
+priority=10
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-storageci
diff --git a/cta-release/oracle-instant-client.repo b/cta-release/oracle-instant-client.repo
new file mode 100644
index 0000000000000000000000000000000000000000..b9a500505b34bd39828cba9dbc6f65209373f70b
--- /dev/null
+++ b/cta-release/oracle-instant-client.repo
@@ -0,0 +1,7 @@
+[oracle-instant-client]
+name=Oracle instant client
+baseurl=https://yum.oracle.com/repo/OracleLinux/OL$releasever/oracle/instantclient/$basearch
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-oracle
+gpgcheck=1
+enabled=1
+priority=1
diff --git a/cta.spec.in b/cta.spec.in
index b963bf21f8241d844c4d62dc2118e51b685124f1..fd91fa999a9155e78d4e9b9b056037780a2dfd4b 100644
--- a/cta.spec.in
+++ b/cta.spec.in
@@ -491,6 +491,27 @@ Currently contains a helper for the client-ar script, which should be installed
%files -n cta-systemtest-helpers
%attr(0755,root,root) /usr/bin/cta-client-ar-abortPrepare
+%package -n cta-release
+Summary: Repository configuration for CTA dependencies
+Group: Application/CTA
+Requires: yum-plugin-versionlock
+%description -n cta-release
+Repository configuration for CTA dependencies
+This package contains .repo files, gpg keys and yum-versionlock configuration fro CTA
+%files -n cta-release
+%defattr(0644,root,root)
+%config(noreplace) %{_sysconfdir}/yum.repos.d/*
+%{_sysconfdir}/pki/rpm-gpg/*
+%{_sysconfdir}/yum/pluginconf.d/versionlock.cta
+
+%post -n cta-release
+cat << EOF
+------
+CTA versionlock file installed as "%{_sysconfdir}/yum/pluginconf.d/versionlock.cta"
+Remember to add its content to "%{_sysconfdir}/yum/pluginconf.d/versionlock.list" to enable it.
+------
+EOF
+
%changelog
* Tue Jul 27 2021 julien.leduc (at) cern.ch - 4.0-5
- [frontend] Add options to "tapepool ls" to filter tapepools on their name, vo and encryption
diff --git a/objectstore/RetrieveRequest.cpp b/objectstore/RetrieveRequest.cpp
index 2960a81115400c8021d0c3bb3f53fe84461ab8da..855667e3c83d37395712e86f335ccb47214b4c7a 100644
--- a/objectstore/RetrieveRequest.cpp
+++ b/objectstore/RetrieveRequest.cpp
@@ -174,8 +174,8 @@ queueForFailure:;
}
// Generate the last failure for this job (tape unavailable).
*j.mutable_failurelogs()->Add() = utils::getCurrentLocalTime() + " " +
- utils::getShortHostname() + " In RetrieveRequest::garbageCollect(): No VID avaiable to requeue the request. Failing it.";
- lc.log(log::ERR, "In RetrieveRequest::garbageCollect(): No VID avaiable to requeue the request. Failing all jobs.");
+ utils::getShortHostname() + " In RetrieveRequest::garbageCollect(): No VID available to requeue the request. Failing it.";
+ lc.log(log::ERR, "In RetrieveRequest::garbageCollect(): No VID available to requeue the request. Failing all jobs.");
}
}
// Ok, the request is ready to be queued. We will queue it to the VID corresponding
diff --git a/xroot_plugins/XrdCtaRecycleTapeFileLs.hpp b/xroot_plugins/XrdCtaRecycleTapeFileLs.hpp
index 7c18034dd8eadefad9f2c83c39860beb5237ab7b..86631b2386d39813e1ae2c644cf76feed24f8d23 100644
--- a/xroot_plugins/XrdCtaRecycleTapeFileLs.hpp
+++ b/xroot_plugins/XrdCtaRecycleTapeFileLs.hpp
@@ -66,15 +66,28 @@ RecycleTapeFileLsStream::RecycleTapeFileLsStream(const RequestMessage &requestMs
searchCriteria.vid = requestMsg.getOptional(OptionString::VID, &has_any);
auto diskFileId = requestMsg.getOptional(OptionString::FXID, &has_any);
+
+ searchCriteria.diskFileIds = requestMsg.getOptional(OptionStrList::FILE_ID, &has_any);
if(diskFileId){
// single option on the command line we need to do the conversion ourselves.
+ if(!searchCriteria.diskFileIds) searchCriteria.diskFileIds = std::vector<std::string>();
+
auto fid = strtol(diskFileId->c_str(), nullptr, 16);
if(fid < 1 || fid == LONG_MAX) {
throw cta::exception::UserError(*diskFileId + " is not a valid file ID");
}
- searchCriteria.diskFileId = std::to_string(fid);
+
+ searchCriteria.diskFileIds->push_back(std::to_string(fid));
}
+ // Disk instance on its own does not give a valid set of search criteria (no &has_any)
+ searchCriteria.diskInstance = requestMsg.getOptional(OptionString::INSTANCE);
+
+ searchCriteria.archiveFileId = requestMsg.getOptional(OptionUInt64::ARCHIVE_FILE_ID, &has_any);
+
+ // Copy number on its own does not give a valid set of search criteria (no &has_any)
+ searchCriteria.copynb = requestMsg.getOptional(OptionUInt64::COPY_NUMBER);
+
if(!has_any){
throw cta::exception::UserError("Must specify at least one search option");
}
diff --git a/xroot_plugins/XrdSsiCtaRequestMessage.cpp b/xroot_plugins/XrdSsiCtaRequestMessage.cpp
index ebf0f37743cf77600792a1b6b437317c5d38df34..5ed7a836baa8e54fe8907454f6823cd9a710e95b 100644
--- a/xroot_plugins/XrdSsiCtaRequestMessage.cpp
+++ b/xroot_plugins/XrdSsiCtaRequestMessage.cpp
@@ -234,6 +234,9 @@ void RequestMessage::process(const cta::xrd::Request &request, cta::xrd::Respons
case cmd_pair(AdminCmd::CMD_TAPEFILE, AdminCmd::SUBCMD_LS):
processTapeFile_Ls(response, stream);
break;
+ case cmd_pair(AdminCmd::CMD_TAPEFILE, AdminCmd::SUBCMD_RM):
+ processTapeFile_Rm(response);
+ break;
case cmd_pair(AdminCmd::CMD_TAPEPOOL, AdminCmd::SUBCMD_ADD):
processTapePool_Add(response);
break;
@@ -1815,6 +1818,29 @@ void RequestMessage::processTapeFile_Ls(cta::xrd::Response &response, XrdSsiStre
response.set_type(cta::xrd::Response::RSP_SUCCESS);
}
+void RequestMessage::processTapeFile_Rm(cta::xrd::Response &response)
+{
+ using namespace cta::admin;
+ auto &vid = getRequired(OptionString::VID);
+ auto &reason = getRequired(OptionString::REASON);
+ auto archiveFileId = getOptional(OptionUInt64::ARCHIVE_FILE_ID);
+ auto instance = getOptional(OptionString::INSTANCE);
+ auto diskFileId = getOptional(OptionString::FXID);
+
+ if (archiveFileId) {
+ m_catalogue.deleteTapeFileCopy(vid, archiveFileId.value(), reason);
+ response.set_type(cta::xrd::Response::RSP_SUCCESS);
+ } else if (diskFileId) {
+ if (!instance) {
+ throw exception::UserError(std::string("--fxid requires that --instance is specified"));
+ }
+ m_catalogue.deleteTapeFileCopy(vid, diskFileId.value(), instance.value(), reason);
+ response.set_type(cta::xrd::Response::RSP_SUCCESS);
+ } else {
+ throw exception::UserError("--id or --fxid must be specified");
+ }
+}
+
void RequestMessage::processTapePool_Add(cta::xrd::Response &response)
{
diff --git a/xroot_plugins/XrdSsiCtaRequestMessage.hpp b/xroot_plugins/XrdSsiCtaRequestMessage.hpp
index 24bf193b2872f6bb74bf26c0636b7dd15cdf9de5..fb9b2f4f27b2eda48cb1a1db3992f2010c1c35e3 100644
--- a/xroot_plugins/XrdSsiCtaRequestMessage.hpp
+++ b/xroot_plugins/XrdSsiCtaRequestMessage.hpp
@@ -198,6 +198,7 @@ private:
void processTape_Rm (cta::xrd::Response &response);
void processTape_Reclaim (cta::xrd::Response &response);
void processTape_Label (cta::xrd::Response &response);
+ void processTapeFile_Rm (cta::xrd::Response &response);
void processTapePool_Add (cta::xrd::Response &response);
void processTapePool_Ch (cta::xrd::Response &response);
void processTapePool_Rm (cta::xrd::Response &response);
@@ -207,7 +208,7 @@ private:
void processVirtualOrganization_Add(cta::xrd::Response &response);
void processVirtualOrganization_Ch(cta::xrd::Response &response);
void processVirtualOrganization_Rm(cta::xrd::Response &response);
-
+
/*!
* Process AdminCmd events which can return a stream response
*