From aa22074ae1ce682442c7279153605e48102cc588 Mon Sep 17 00:00:00 2001
From: Cedric Caffy <cedric.caffy@cern.ch>
Date: Thu, 3 Dec 2020 17:49:31 +0100
Subject: [PATCH] [catalogue] Changed the reclaim logic + file deletion logic
---
catalogue/Catalogue.hpp | 7 +-
catalogue/CatalogueRetryWrapper.hpp | 4 +-
catalogue/CatalogueTest.cpp | 55 ++-
catalogue/DummyCatalogue.hpp | 2 +-
catalogue/InsertFileRecycleLog.hpp | 8 +-
catalogue/MysqlCatalogue.cpp | 4 +-
catalogue/MysqlCatalogue.hpp | 6 +-
catalogue/OracleCatalogue.cpp | 73 +---
catalogue/OracleCatalogue.hpp | 6 +-
catalogue/PostgresCatalogue.cpp | 74 +---
catalogue/PostgresCatalogue.hpp | 6 +-
catalogue/RdbmsCatalogue.cpp | 329 +++++++-----------
catalogue/RdbmsCatalogue.hpp | 50 ++-
catalogue/SqliteCatalogue.cpp | 4 +-
catalogue/SqliteCatalogue.hpp | 6 +-
.../dataStructures/DeleteArchiveRequest.hpp | 3 +
scheduler/Scheduler.cpp | 2 +-
xroot_plugins/XrdSsiCtaRequestMessage.cpp | 15 +-
18 files changed, 242 insertions(+), 412 deletions(-)
diff --git a/catalogue/Catalogue.hpp b/catalogue/Catalogue.hpp
index 9a54c7be53..04de8f7e67 100644
--- a/catalogue/Catalogue.hpp
+++ b/catalogue/Catalogue.hpp
@@ -1024,12 +1024,13 @@ public:
const std::string &diskFileId) = 0;
/**
- * Insert the tape files and ArchiveFiles entries in the recycle-bin and delete
- * them from the TAPE_FILE and ARCHIVE_FILE tables
+ * Insert the ArchiveFile and all its tape files in the FILE_RECYCLE_LOG table.
+ * There will be one entry on the FILE_RECYCLE_LOG table per deleted tape file
+ *
* @param request the DeleteRequest object that holds information about the file to delete.
* @param lc the logContext
*/
- virtual void moveArchiveFileToRecycleBin(const common::dataStructures::DeleteArchiveRequest &request,
+ virtual void moveArchiveFileToRecycleLog(const common::dataStructures::DeleteArchiveRequest &request,
log::LogContext & lc) = 0;
/**
diff --git a/catalogue/CatalogueRetryWrapper.hpp b/catalogue/CatalogueRetryWrapper.hpp
index 9d1502e9f2..3bef9b3b82 100644
--- a/catalogue/CatalogueRetryWrapper.hpp
+++ b/catalogue/CatalogueRetryWrapper.hpp
@@ -598,9 +598,9 @@ public:
return retryOnLostConnection(m_log, [&]{return m_catalogue->updateDiskFileId(archiveFileId, diskInstance, diskFileId);}, m_maxTriesToConnect);
}
- void moveArchiveFileToRecycleBin(const common::dataStructures::DeleteArchiveRequest &request,
+ void moveArchiveFileToRecycleLog(const common::dataStructures::DeleteArchiveRequest &request,
log::LogContext & lc) override {
- return retryOnLostConnection(m_log,[&]{return m_catalogue->moveArchiveFileToRecycleBin(request,lc);},m_maxTriesToConnect);
+ return retryOnLostConnection(m_log,[&]{return m_catalogue->moveArchiveFileToRecycleLog(request,lc);},m_maxTriesToConnect);
}
protected:
diff --git a/catalogue/CatalogueTest.cpp b/catalogue/CatalogueTest.cpp
index 1f9d9ba561..221e6c8739 100644
--- a/catalogue/CatalogueTest.cpp
+++ b/catalogue/CatalogueTest.cpp
@@ -9168,6 +9168,8 @@ TEST_P(cta_catalogue_CatalogueTest, prepareToRetrieveFileUsingArchiveFileId_retu
file2Written.copyNb = 1;
file2Written.tapeDrive = tapeDrive;
m_catalogue->filesWrittenToTape(file2WrittenSet);
+
+ ASSERT_TRUE(m_catalogue->getFileRecycleLogItor().hasMore());
const std::string mountPolicyName = "mount_policy";
const uint64_t archivePriority = 1;
@@ -15721,13 +15723,14 @@ TEST_P(cta_catalogue_CatalogueTest, moveFilesToRecycleBin) {
req.diskFileId = tapeItem->diskFileId;
req.diskFilePath = tapeItem->diskFilePath;
req.diskInstance = tapeItem->diskInstance;
- ASSERT_NO_THROW(m_catalogue->moveArchiveFileToRecycleBin(req,dummyLc));
+ req.archiveFile = m_catalogue->getArchiveFileById(tapeItem->archiveFileId);
+ ASSERT_NO_THROW(m_catalogue->moveArchiveFileToRecycleLog(req,dummyLc));
}
ASSERT_FALSE(m_catalogue->getArchiveFilesItor().hasMore());
- std::vector<common::dataStructures::DeletedArchiveFile> deletedArchiveFiles;
+ std::vector<common::dataStructures::FileRecycleLog> deletedArchiveFiles;
{
- auto itor = m_catalogue->getDeletedArchiveFilesItor();
+ auto itor = m_catalogue->getFileRecycleLogItor();
while(itor.hasMore()){
deletedArchiveFiles.push_back(itor.next());
}
@@ -15747,40 +15750,36 @@ TEST_P(cta_catalogue_CatalogueTest, moveFilesToRecycleBin) {
std::ostringstream diskFilePath;
diskFilePath << "/test/file"<<i;
- ASSERT_EQ(i,deletedArchiveFile.archiveFileID);
- ASSERT_EQ(diskInstance,deletedArchiveFile.diskInstance);
+ ASSERT_EQ(i,deletedArchiveFile.archiveFileId);
+ ASSERT_EQ(diskInstance,deletedArchiveFile.diskInstanceName);
ASSERT_EQ(diskFileId.str(),deletedArchiveFile.diskFileId);
ASSERT_EQ(diskFilePath.str(),deletedArchiveFile.diskFilePath);
- ASSERT_EQ(PUBLIC_DISK_USER,deletedArchiveFile.diskFileInfo.owner_uid);
- ASSERT_EQ(PUBLIC_DISK_GROUP,deletedArchiveFile.diskFileInfo.gid);
- ASSERT_EQ(archiveFileSize,deletedArchiveFile.fileSize);
+ ASSERT_EQ(PUBLIC_DISK_USER,deletedArchiveFile.diskFileUid);
+ ASSERT_EQ(PUBLIC_DISK_GROUP,deletedArchiveFile.diskFileGid);
+ ASSERT_EQ(archiveFileSize,deletedArchiveFile.sizeInBytes);
ASSERT_EQ(cta::checksum::ChecksumBlob(checksum::ADLER32, "1357"),deletedArchiveFile.checksumBlob);
- ASSERT_EQ(m_storageClassSingleCopy.name, deletedArchiveFile.storageClass);
+ ASSERT_EQ(m_storageClassSingleCopy.name, deletedArchiveFile.storageClassName);
ASSERT_EQ(diskFileId.str(),deletedArchiveFile.diskFileIdWhenDeleted);
- auto tapeFile = deletedArchiveFile.tapeFiles.at(1);
- ASSERT_EQ(tape1.vid, tapeFile.vid);
- ASSERT_EQ(i,tapeFile.fSeq);
- ASSERT_EQ(i * 100,tapeFile.blockId);
- ASSERT_EQ(1, tapeFile.copyNb);
- ASSERT_EQ(archiveFileSize,tapeFile.fileSize);
+ ASSERT_EQ(tape1.vid, deletedArchiveFile.vid);
+ ASSERT_EQ(i,deletedArchiveFile.fSeq);
+ ASSERT_EQ(i * 100,deletedArchiveFile.blockId);
+ ASSERT_EQ(1, deletedArchiveFile.copyNb);
+ ASSERT_EQ(archiveFileSize,deletedArchiveFile.logicalSizeInBytes);
}
//Let's try the deletion of the files from the recycle-bin.
for(uint64_t i = 1; i <= nbArchiveFiles; i++) {
- m_catalogue->deleteFileFromRecycleBin(i,dummyLc);
+ m_catalogue->deleteFilesFromRecycleLog(tape1.vid,dummyLc);
}
{
- auto itor = m_catalogue->getDeletedArchiveFilesItor();
+ auto itor = m_catalogue->getFileRecycleLogItor();
ASSERT_FALSE(itor.hasMore());
}
- //Delete an archive file from the recycle-bin should be idempotent
- ASSERT_NO_THROW(m_catalogue->deleteFileFromRecycleBin(12532,dummyLc));
-
}
-TEST_P(cta_catalogue_CatalogueTest, reclaimTapeRemovesFilesFromRecycleBin) {
+TEST_P(cta_catalogue_CatalogueTest, reclaimTapeRemovesFilesFromRecycleLog) {
using namespace cta;
const bool logicalLibraryIsDisabled= false;
@@ -15851,27 +15850,27 @@ TEST_P(cta_catalogue_CatalogueTest, reclaimTapeRemovesFilesFromRecycleBin) {
req.diskFileId = tapeItem->diskFileId;
req.diskFilePath = tapeItem->diskFilePath;
req.diskInstance = tapeItem->diskInstance;
- ASSERT_NO_THROW(m_catalogue->moveArchiveFileToRecycleBin(req,dummyLc));
+ req.archiveFile = m_catalogue->getArchiveFileById(tapeItem->archiveFileId);
+ ASSERT_NO_THROW(m_catalogue->moveArchiveFileToRecycleLog(req,dummyLc));
}
ASSERT_FALSE(m_catalogue->getArchiveFilesItor().hasMore());
-
- std::vector<common::dataStructures::DeletedArchiveFile> deletedArchiveFiles;
+ std::vector<common::dataStructures::FileRecycleLog> deletedArchiveFiles;
{
- auto itor = m_catalogue->getDeletedArchiveFilesItor();
+ auto itor = m_catalogue->getFileRecycleLogItor();
while(itor.hasMore()){
deletedArchiveFiles.push_back(itor.next());
}
}
- //And test that these files are there.
- //Run the unit test for all the databases
+ //And test that these files are in the recycle log
ASSERT_EQ(nbArchiveFiles,deletedArchiveFiles.size());
+ ASSERT_TRUE(m_catalogue->getFileRecycleLogItor().hasMore());
//Reclaim the tape
m_catalogue->setTapeFull(m_admin, tape1.vid, true);
m_catalogue->reclaimTape(m_admin, tape1.vid, dummyLc);
{
- auto itor = m_catalogue->getDeletedArchiveFilesItor();
+ auto itor = m_catalogue->getFileRecycleLogItor();
ASSERT_FALSE(itor.hasMore());
}
}
diff --git a/catalogue/DummyCatalogue.hpp b/catalogue/DummyCatalogue.hpp
index ee988e747b..8d650dd012 100644
--- a/catalogue/DummyCatalogue.hpp
+++ b/catalogue/DummyCatalogue.hpp
@@ -164,7 +164,7 @@ public:
void tapeMountedForRetrieve(const std::string& vid, const std::string& drive) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
bool tapePoolExists(const std::string& tapePoolName) const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
void updateDiskFileId(uint64_t archiveFileId, const std::string &diskInstance, const std::string &diskFileId) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
- void moveArchiveFileToRecycleBin(const common::dataStructures::DeleteArchiveRequest &request,
+ void moveArchiveFileToRecycleLog(const common::dataStructures::DeleteArchiveRequest &request,
log::LogContext & lc) override {throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented");}
// Special functions for unit tests.
void addEnabledTape(const std::string & vid) {
diff --git a/catalogue/InsertFileRecycleLog.hpp b/catalogue/InsertFileRecycleLog.hpp
index f7cc50d4fc..51d3f5bc87 100644
--- a/catalogue/InsertFileRecycleLog.hpp
+++ b/catalogue/InsertFileRecycleLog.hpp
@@ -28,19 +28,23 @@ namespace cta { namespace catalogue {
class InsertFileRecycleLog {
public:
std::string vid;
- uint64_t fseq;
+ uint64_t fSeq;
uint64_t blockId;
uint64_t logicalSizeInBytes;
uint8_t copyNb;
time_t tapeFileCreationTime;
uint64_t archiveFileId;
- std::string diskFilePath;
+ cta::optional<std::string> diskFilePath;
std::string reasonLog;
time_t recycleLogTime;
static std::string getRepackReasonLog(){
return "REPACK";
}
+
+ static std::string getDeletionReasonLog(const std::string & deleterName, const std::string & diskInstanceName){
+ return "File deleted by " + deleterName + "from the " + diskInstanceName + "instance";
+ }
};
}}
diff --git a/catalogue/MysqlCatalogue.cpp b/catalogue/MysqlCatalogue.cpp
index 00721767bd..c5bc490ed4 100644
--- a/catalogue/MysqlCatalogue.cpp
+++ b/catalogue/MysqlCatalogue.cpp
@@ -729,14 +729,14 @@ void MysqlCatalogue::DO_NOT_USE_deleteArchiveFile_DO_NOT_USE(const std::string &
//------------------------------------------------------------------------------
// copyArchiveFileToRecycleBinAndDelete
//------------------------------------------------------------------------------
-void MysqlCatalogue::copyArchiveFileToRecycleBinAndDelete(rdbms::Conn & conn, const common::dataStructures::DeleteArchiveRequest &request, log::LogContext & lc){
+void MysqlCatalogue::copyArchiveFileToFileRecyleLogAndDelete(rdbms::Conn & conn, const common::dataStructures::DeleteArchiveRequest &request, log::LogContext & lc){
try {
utils::Timer t;
log::TimingList tl;
//We currently do an INSERT INTO and a DELETE FROM
//in a single transaction
conn.executeNonQuery("START TRANSACTION");
- copyArchiveFileToRecycleBin(conn,request);
+ copyArchiveFileToFileRecycleLog(conn,request);
tl.insertAndReset("insertToRecycleBinTime",t);
setTapeDirty(conn,request.archiveFileID);
tl.insertAndReset("setTapeDirtyTime",t);
diff --git a/catalogue/MysqlCatalogue.hpp b/catalogue/MysqlCatalogue.hpp
index 15fc098b23..955b427205 100644
--- a/catalogue/MysqlCatalogue.hpp
+++ b/catalogue/MysqlCatalogue.hpp
@@ -194,13 +194,13 @@ protected:
log::LogContext &lc) override;
/**
- * Copy the archiveFile and the associated tape files from the ARCHIVE_FILE and TAPE_FILE tables to the recycle-bin tables
+ * Copy the archiveFile and the associated tape files from the ARCHIVE_FILE and TAPE_FILE tables to the FILE_RECYCLE_LOG table
* and deletes the ARCHIVE_FILE and TAPE_FILE entries.
* @param conn the database connection
- * @param request the request that contains the necessary informations to identify the archiveFile to copy to the recycle-bin
+ * @param request the request that contains the necessary informations to identify the archiveFile to copy to the FILE_RECYCLE_LOG table
* @param lc the log context
*/
- void copyArchiveFileToRecycleBinAndDelete(rdbms::Conn & conn,const common::dataStructures::DeleteArchiveRequest &request, log::LogContext & lc) override;
+ void copyArchiveFileToFileRecyleLogAndDelete(rdbms::Conn & conn,const common::dataStructures::DeleteArchiveRequest &request, log::LogContext & lc) override;
/**
* Delete the TapeFiles and the ArchiveFile from the recycle-bin in one transaction
diff --git a/catalogue/OracleCatalogue.cpp b/catalogue/OracleCatalogue.cpp
index c852a188da..4acede6852 100644
--- a/catalogue/OracleCatalogue.cpp
+++ b/catalogue/OracleCatalogue.cpp
@@ -593,7 +593,7 @@ void OracleCatalogue::filesWrittenToTape(const std::set<TapeItemWrittenPointer>
auto stmt = conn.createStmt(sql);
stmt.bindString(":VID",recycledFile.vid);
- stmt.bindUint64(":FSEQ",recycledFile.fseq);
+ stmt.bindUint64(":FSEQ",recycledFile.fSeq);
stmt.executeNonQuery();
}
@@ -788,7 +788,7 @@ std::list<cta::catalogue::InsertFileRecycleLog> OracleCatalogue::insertOldCopies
while(rset.next()){
cta::catalogue::InsertFileRecycleLog fileRecycleLog;
fileRecycleLog.vid = rset.columnString("VID");
- fileRecycleLog.fseq = rset.columnUint64("FSEQ");
+ fileRecycleLog.fSeq = rset.columnUint64("FSEQ");
fileRecycleLog.blockId = rset.columnUint64("BLOCK_ID");
fileRecycleLog.logicalSizeInBytes = rset.columnUint64("LOGICAL_SIZE_IN_BYTES");
fileRecycleLog.copyNb = rset.columnUint8("COPY_NB");
@@ -801,70 +801,7 @@ std::list<cta::catalogue::InsertFileRecycleLog> OracleCatalogue::insertOldCopies
}
{
for(auto & fileRecycleLog: fileRecycleLogsToInsert){
- const char *const sql =
- "INSERT INTO FILE_RECYCLE_LOG("
- "FILE_RECYCLE_LOG_ID,"
- "VID,"
- "FSEQ,"
- "BLOCK_ID,"
- "LOGICAL_SIZE_IN_BYTES,"
- "COPY_NB,"
- "TAPE_FILE_CREATION_TIME,"
- "ARCHIVE_FILE_ID,"
- "DISK_INSTANCE_NAME,"
- "DISK_FILE_ID,"
- "DISK_FILE_ID_WHEN_DELETED,"
- "DISK_FILE_UID,"
- "DISK_FILE_GID,"
- "SIZE_IN_BYTES,"
- "CHECKSUM_BLOB,"
- "CHECKSUM_ADLER32,"
- "STORAGE_CLASS_ID,"
- "ARCHIVE_FILE_CREATION_TIME,"
- "RECONCILIATION_TIME,"
- "COLLOCATION_HINT,"
- "REASON_LOG,"
- "RECYCLE_LOG_TIME"
- ") SELECT "
- "FILE_RECYCLE_LOG_ID_SEQ.NEXTVAL,"
- ":VID,"
- ":FSEQ,"
- ":BLOCK_ID,"
- ":LOGICAL_SIZE_IN_BYTES,"
- ":COPY_NB,"
- ":TAPE_FILE_CREATION_TIME,"
- ":ARCHIVE_FILE_ID,"
- "ARCHIVE_FILE.DISK_INSTANCE_NAME AS DISK_INSTANCE_NAME,"
- "ARCHIVE_FILE.DISK_FILE_ID AS DISK_FILE_ID,"
- "ARCHIVE_FILE.DISK_FILE_ID AS DISK_FILE_ID_2,"
- "ARCHIVE_FILE.DISK_FILE_UID AS DISK_FILE_UID,"
- "ARCHIVE_FILE.DISK_FILE_GID AS DISK_FILE_GID,"
- "ARCHIVE_FILE.SIZE_IN_BYTES AS SIZE_IN_BYTES,"
- "ARCHIVE_FILE.CHECKSUM_BLOB AS CHECKSUM_BLOB,"
- "ARCHIVE_FILE.CHECKSUM_ADLER32 AS CHECKSUM_ADLER32,"
- "ARCHIVE_FILE.STORAGE_CLASS_ID AS STORAGE_CLASS_ID,"
- "ARCHIVE_FILE.CREATION_TIME AS ARCHIVE_FILE_CREATION_TIME,"
- "ARCHIVE_FILE.RECONCILIATION_TIME AS RECONCILIATION_TIME,"
- "ARCHIVE_FILE.COLLOCATION_HINT AS COLLOCATION_HINT,"
- ":REASON_LOG,"
- ":RECYCLE_LOG_TIME "
- "FROM "
- "DUAL,"
- "ARCHIVE_FILE "
- "WHERE "
- "ARCHIVE_FILE.ARCHIVE_FILE_ID = :ARCHIVE_FILE_ID_2";
- auto stmt = conn.createStmt(sql);
- stmt.bindString(":VID",fileRecycleLog.vid);
- stmt.bindUint64(":FSEQ",fileRecycleLog.fseq);
- stmt.bindUint64(":BLOCK_ID",fileRecycleLog.blockId);
- stmt.bindUint64(":LOGICAL_SIZE_IN_BYTES",fileRecycleLog.logicalSizeInBytes);
- stmt.bindUint8(":COPY_NB",fileRecycleLog.copyNb);
- stmt.bindUint64(":TAPE_FILE_CREATION_TIME",fileRecycleLog.tapeFileCreationTime);
- stmt.bindUint64(":ARCHIVE_FILE_ID",fileRecycleLog.archiveFileId);
- stmt.bindString(":REASON_LOG",fileRecycleLog.reasonLog);
- stmt.bindUint64(":RECYCLE_LOG_TIME",fileRecycleLog.recycleLogTime);
- stmt.bindUint64(":ARCHIVE_FILE_ID_2",fileRecycleLog.archiveFileId);
- stmt.executeNonQuery();
+ insertFileInFileRecycleLog(conn,fileRecycleLog);
}
}
return fileRecycleLogsToInsert;
@@ -1123,14 +1060,14 @@ void OracleCatalogue::DO_NOT_USE_deleteArchiveFile_DO_NOT_USE(const std::string
//------------------------------------------------------------------------------
// copyArchiveFileToRecycleBinAndDelete
//------------------------------------------------------------------------------
-void OracleCatalogue::copyArchiveFileToRecycleBinAndDelete(rdbms::Conn & conn, const common::dataStructures::DeleteArchiveRequest &request, log::LogContext & lc){
+void OracleCatalogue::copyArchiveFileToFileRecyleLogAndDelete(rdbms::Conn & conn, const common::dataStructures::DeleteArchiveRequest &request, log::LogContext & lc){
try {
utils::Timer t;
log::TimingList tl;
//We currently do an INSERT INTO, update and two DELETE FROM
//in a single transaction
conn.setAutocommitMode(rdbms::AutocommitMode::AUTOCOMMIT_OFF);
- copyArchiveFileToRecycleBin(conn,request);
+ copyArchiveFileToFileRecycleLog(conn,request);
tl.insertAndReset("insertToRecycleBinTime",t);
setTapeDirty(conn,request.archiveFileID);
tl.insertAndReset("setTapeDirtyTime",t);
diff --git a/catalogue/OracleCatalogue.hpp b/catalogue/OracleCatalogue.hpp
index f51bd9a10f..74f311dac1 100644
--- a/catalogue/OracleCatalogue.hpp
+++ b/catalogue/OracleCatalogue.hpp
@@ -237,13 +237,13 @@ private:
std::list<cta::catalogue::InsertFileRecycleLog> insertOldCopiesOfFilesIfAnyOnFileRecycleLog(rdbms::Conn & conn);
/**
- * Copy the archiveFile and the associated tape files from the ARCHIVE_FILE and TAPE_FILE tables to the recycle-bin tables
+ * Copy the archiveFile and the associated tape files from the ARCHIVE_FILE and TAPE_FILE tables to the FILE_RECYCLE_LOG table
* and deletes the ARCHIVE_FILE and TAPE_FILE entries.
* @param conn the database connection
- * @param request the request that contains the necessary informations to identify the archiveFile to copy to the recycle-bin
+ * @param request the request that contains the necessary informations to identify the archiveFile to copy to the FILE_RECYCLE_LOG table
* @param lc the log context
*/
- void copyArchiveFileToRecycleBinAndDelete(rdbms::Conn & conn, const common::dataStructures::DeleteArchiveRequest &request, log::LogContext & lc) override;
+ void copyArchiveFileToFileRecyleLogAndDelete(rdbms::Conn & conn, const common::dataStructures::DeleteArchiveRequest &request, log::LogContext & lc) override;
/**
* Delete the TapeFiles and the ArchiveFile from the recycle-bin in one transaction
diff --git a/catalogue/PostgresCatalogue.cpp b/catalogue/PostgresCatalogue.cpp
index 1dfb9d9df5..00c66d5a7a 100644
--- a/catalogue/PostgresCatalogue.cpp
+++ b/catalogue/PostgresCatalogue.cpp
@@ -533,7 +533,7 @@ void PostgresCatalogue::filesWrittenToTape(const std::set<TapeItemWrittenPointer
"DELETE FROM TAPE_FILE WHERE TAPE_FILE.VID = :VID AND TAPE_FILE.FSEQ = :FSEQ";
auto deleteTapeFileStmt = conn.createStmt(deleteTapeFileSql);
deleteTapeFileStmt.bindString(":VID",recycledFile.vid);
- deleteTapeFileStmt.bindUint64(":FSEQ",recycledFile.fseq);
+ deleteTapeFileStmt.bindUint64(":FSEQ",recycledFile.fSeq);
deleteTapeFileStmt.executeNonQuery();
}
@@ -697,7 +697,7 @@ std::list<cta::catalogue::InsertFileRecycleLog> PostgresCatalogue::insertOldCopi
while(rset.next()){
cta::catalogue::InsertFileRecycleLog fileRecycleLog;
fileRecycleLog.vid = rset.columnString("VID");
- fileRecycleLog.fseq = rset.columnUint64("FSEQ");
+ fileRecycleLog.fSeq = rset.columnUint64("FSEQ");
fileRecycleLog.blockId = rset.columnUint64("BLOCK_ID");
fileRecycleLog.logicalSizeInBytes = rset.columnUint64("LOGICAL_SIZE_IN_BYTES");
fileRecycleLog.copyNb = rset.columnUint8("COPY_NB");
@@ -710,72 +710,10 @@ std::list<cta::catalogue::InsertFileRecycleLog> PostgresCatalogue::insertOldCopi
}
{
for(auto & fileRecycleLog: fileRecycleLogsToInsert){
- const char *const sql =
- "INSERT INTO FILE_RECYCLE_LOG("
- "FILE_RECYCLE_LOG_ID,"
- "VID,"
- "FSEQ,"
- "BLOCK_ID,"
- "LOGICAL_SIZE_IN_BYTES,"
- "COPY_NB,"
- "TAPE_FILE_CREATION_TIME,"
- "ARCHIVE_FILE_ID,"
- "DISK_INSTANCE_NAME,"
- "DISK_FILE_ID,"
- "DISK_FILE_ID_WHEN_DELETED,"
- "DISK_FILE_UID,"
- "DISK_FILE_GID,"
- "SIZE_IN_BYTES,"
- "CHECKSUM_BLOB,"
- "CHECKSUM_ADLER32,"
- "STORAGE_CLASS_ID,"
- "ARCHIVE_FILE_CREATION_TIME,"
- "RECONCILIATION_TIME,"
- "COLLOCATION_HINT,"
- "REASON_LOG,"
- "RECYCLE_LOG_TIME"
- ") SELECT "
- "nextval('FILE_RECYCLE_LOG_ID_SEQ'),"
- ":VID,"
- ":FSEQ,"
- ":BLOCK_ID,"
- ":LOGICAL_SIZE_IN_BYTES,"
- ":COPY_NB,"
- ":TAPE_FILE_CREATION_TIME,"
- ":ARCHIVE_FILE_ID,"
- "ARCHIVE_FILE.DISK_INSTANCE_NAME,"
- "ARCHIVE_FILE.DISK_FILE_ID,"
- "ARCHIVE_FILE.DISK_FILE_ID,"
- "ARCHIVE_FILE.DISK_FILE_UID,"
- "ARCHIVE_FILE.DISK_FILE_GID,"
- "ARCHIVE_FILE.SIZE_IN_BYTES,"
- "ARCHIVE_FILE.CHECKSUM_BLOB,"
- "ARCHIVE_FILE.CHECKSUM_ADLER32,"
- "ARCHIVE_FILE.STORAGE_CLASS_ID,"
- "ARCHIVE_FILE.CREATION_TIME,"
- "ARCHIVE_FILE.RECONCILIATION_TIME,"
- "ARCHIVE_FILE.COLLOCATION_HINT,"
- ":REASON_LOG,"
- ":RECYCLE_LOG_TIME "
- "FROM "
- "ARCHIVE_FILE "
- "WHERE "
- "ARCHIVE_FILE.ARCHIVE_FILE_ID = :ARCHIVE_FILE_ID_2";
- auto stmt = conn.createStmt(sql);
- stmt.bindString(":VID",fileRecycleLog.vid);
- stmt.bindUint64(":FSEQ",fileRecycleLog.fseq);
- stmt.bindUint64(":BLOCK_ID",fileRecycleLog.blockId);
- stmt.bindUint64(":LOGICAL_SIZE_IN_BYTES",fileRecycleLog.logicalSizeInBytes);
- stmt.bindUint8(":COPY_NB",fileRecycleLog.copyNb);
- stmt.bindUint64(":TAPE_FILE_CREATION_TIME",fileRecycleLog.tapeFileCreationTime);
- stmt.bindUint64(":ARCHIVE_FILE_ID",fileRecycleLog.archiveFileId);
- stmt.bindString(":REASON_LOG",fileRecycleLog.reasonLog);
- stmt.bindUint64(":RECYCLE_LOG_TIME",fileRecycleLog.recycleLogTime);
- stmt.bindUint64(":ARCHIVE_FILE_ID_2",fileRecycleLog.archiveFileId);
- stmt.executeNonQuery();
+ insertFileInFileRecycleLog(conn,fileRecycleLog);
}
+ return fileRecycleLogsToInsert;
}
- return fileRecycleLogsToInsert;
} catch(exception::Exception &ex) {
ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str());
throw;
@@ -1080,14 +1018,14 @@ void PostgresCatalogue::beginCreateTemporarySetDeferred(rdbms::Conn &conn) const
//------------------------------------------------------------------------------
// copyArchiveFileToRecycleBinAndDelete
//------------------------------------------------------------------------------
-void PostgresCatalogue::copyArchiveFileToRecycleBinAndDelete(rdbms::Conn & conn, const common::dataStructures::DeleteArchiveRequest &request, log::LogContext & lc) {
+void PostgresCatalogue::copyArchiveFileToFileRecyleLogAndDelete(rdbms::Conn & conn, const common::dataStructures::DeleteArchiveRequest &request, log::LogContext & lc) {
try {
utils::Timer t;
log::TimingList tl;
//We currently do an INSERT INTO and a DELETE FROM
//in a single transaction
conn.executeNonQuery("BEGIN");
- copyArchiveFileToRecycleBin(conn,request);
+ copyArchiveFileToFileRecycleLog(conn,request);
tl.insertAndReset("insertToRecycleBinTime",t);
setTapeDirty(conn,request.archiveFileID);
tl.insertAndReset("setTapeDirtyTime",t);
diff --git a/catalogue/PostgresCatalogue.hpp b/catalogue/PostgresCatalogue.hpp
index 5a216a5dca..93e284b35f 100644
--- a/catalogue/PostgresCatalogue.hpp
+++ b/catalogue/PostgresCatalogue.hpp
@@ -271,13 +271,13 @@ private:
void insertTapeFileBatchIntoTempTable(rdbms::Conn &conn, const std::set<TapeFileWritten> &events) const;
/**
- * Copy the archiveFile and the associated tape files from the ARCHIVE_FILE and TAPE_FILE tables to the recycle-bin tables
+ * Copy the archiveFile and the associated tape files from the ARCHIVE_FILE and TAPE_FILE tables to the FILE_RECYCLE_LOG table
* and deletes the ARCHIVE_FILE and TAPE_FILE entries.
* @param conn the database connection
- * @param request the request that contains the necessary informations to identify the archiveFile to copy to the recycle-bin
+ * @param request the request that contains the necessary informations to identify the archiveFile to copy to the FILE_RECYCLE_LOG table
* @param lc the log context
*/
- void copyArchiveFileToRecycleBinAndDelete(rdbms::Conn & conn, const common::dataStructures::DeleteArchiveRequest &request, log::LogContext & lc) override;
+ void copyArchiveFileToFileRecyleLogAndDelete(rdbms::Conn & conn, const common::dataStructures::DeleteArchiveRequest &request, log::LogContext & lc) override;
/**
* Delete the TapeFiles and the ArchiveFile from the recycle-bin in one transaction
diff --git a/catalogue/RdbmsCatalogue.cpp b/catalogue/RdbmsCatalogue.cpp
index d7edf975fb..8861bdeddb 100644
--- a/catalogue/RdbmsCatalogue.cpp
+++ b/catalogue/RdbmsCatalogue.cpp
@@ -3910,28 +3910,6 @@ std::map<std::string, std::string> &vidToLogicalLibrary) const {
}
}
-//------------------------------------------------------------------------------
-//getNbNonSupersededFilesOnTape
-//------------------------------------------------------------------------------
-uint64_t RdbmsCatalogue::getNbNonSupersededFilesOnTape(rdbms::Conn& conn, const std::string& vid) const {
- try {
- const char *const sql =
- "SELECT COUNT(*) AS NB_NON_SUPERSEDED_FILES FROM TAPE_FILE "
- "WHERE VID = :VID "
- "AND SUPERSEDED_BY_VID IS NULL "
- "AND SUPERSEDED_BY_FSEQ IS NULL";
- auto stmt = conn.createStmt(sql);
- stmt.bindString(":VID", vid);
- auto rset = stmt.executeQuery();
- rset.next();
- return rset.columnUint64("NB_NON_SUPERSEDED_FILES");
- } catch(exception::Exception &ex) {
- ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str());
- throw;
- }
-}
-
-
//------------------------------------------------------------------------------
// getNbFilesOnTape
//------------------------------------------------------------------------------
@@ -4078,13 +4056,11 @@ void RdbmsCatalogue::reclaimTape(const common::dataStructures::SecurityIdentity
}
}
//The tape exists and is full, we can try to reclaim it
- if(getNbNonSupersededFilesOnTape(conn,vid) == 0){
- tl.insertAndReset("getNbNonSupersededFilesOnTapeTime",t);
- //There is no non-superseded files on the tape, we can reclaim it : delete the files and reset the counters
- deleteTapeFiles(conn,vid);
- tl.insertAndReset("deleteTapeFilesTime",t);
- deleteFilesFromRecycleBin(conn,vid,lc);
- tl.insertAndReset("deleteFileFromRecycleBinTime",t);
+ if(this->getNbFilesOnTape(conn,vid) == 0){
+ tl.insertAndReset("getNbFilesOnTape",t);
+ //There is no files on the tape, we can reclaim it : delete the files and reset the counters
+ deleteFilesFromRecycleLog(conn,vid,lc);
+ tl.insertAndReset("deleteFileFromRecycleLogTime",t);
resetTapeCounters(conn,admin,vid);
tl.insertAndReset("resetTapeCountersTime",t);
log::ScopedParamContainer spc(lc);
@@ -7734,7 +7710,7 @@ void RdbmsCatalogue::insertTapeFile(
"FSEQ=:FSEQ";
auto stmt = conn.createStmt(sql);
stmt.bindString(":VID",fileRecycleLog.vid);
- stmt.bindUint64(":FSEQ",fileRecycleLog.fseq);
+ stmt.bindUint64(":FSEQ",fileRecycleLog.fSeq);
stmt.executeNonQuery();
}
}
@@ -8476,20 +8452,15 @@ void RdbmsCatalogue::updateDiskFileId(const uint64_t archiveFileId, const std::s
//------------------------------------------------------------------------------
// moveArchiveFileToRecycleBin
//------------------------------------------------------------------------------
-void RdbmsCatalogue::moveArchiveFileToRecycleBin(const common::dataStructures::DeleteArchiveRequest &request,
+void RdbmsCatalogue::moveArchiveFileToRecycleLog(const common::dataStructures::DeleteArchiveRequest &request,
log::LogContext & lc) {
- cta::common::dataStructures::ArchiveFile archiveFile;
- utils::Timer t, totalTime;
- log::TimingList tl;
- try{
- archiveFile = getArchiveFileById(request.archiveFileID);
- tl.insertAndReset("getArchiveFileByIdTime",t);
- } catch (cta::exception::Exception &ex){
- log::ScopedParamContainer spc(lc);
- spc.add("fileId", request.archiveFileID);
- lc.log(log::WARNING, "Ignoring request to delete archive file because it does not exist in the catalogue");
+ if(!request.archiveFile){
+ //The archive file does not exist in the catalogue, nothing to do with it
return;
}
+ cta::common::dataStructures::ArchiveFile archiveFile = request.archiveFile.value();
+ utils::Timer t, totalTime;
+ log::TimingList tl;
try {
checkDeleteRequestConsistency(request,archiveFile);
tl.insertAndReset("checkDeleteRequestConsistency",t);
@@ -8518,10 +8489,10 @@ void RdbmsCatalogue::moveArchiveFileToRecycleBin(const common::dataStructures::D
<< " fileSize: " << it->fileSize;
spc.add("TAPE FILE", tapeCopyLogStream.str());
}
- lc.log(log::WARNING, "Failed to move archive file to recycle-bin.");
+ lc.log(log::WARNING, "Failed to move archive file to the file-recycle-log.");
exception::UserError ue;
- ue.getMessage() << "Failed to move archive file with ID " << request.archiveFileID << " to the recycle-bin. errorMessage=" << ex.getMessageValue();
+ ue.getMessage() << "Failed to move archive file with ID " << request.archiveFileID << " to the file-recycle-log. errorMessage=" << ex.getMessageValue();
throw ue;
}
@@ -8529,8 +8500,8 @@ void RdbmsCatalogue::moveArchiveFileToRecycleBin(const common::dataStructures::D
//All checks are good, we can move the file to the recycle-bin
auto conn = m_connPool.getConn();
rdbms::AutoRollback autoRollback(conn);
- copyArchiveFileToRecycleBinAndDelete(conn,request,lc);
- tl.insertAndReset("copyArchiveFileToRecycleBinAndDeleteTime",t);
+ copyArchiveFileToFileRecyleLogAndDelete(conn,request,lc);
+ tl.insertAndReset("copyArchiveFileToFileRecyleLogAndDeleteTime",t);
tl.insertAndReset("totalTime",totalTime);
log::ScopedParamContainer spc(lc);
spc.add("fileId", std::to_string(request.archiveFileID))
@@ -8555,7 +8526,7 @@ void RdbmsCatalogue::moveArchiveFileToRecycleBin(const common::dataStructures::D
spc.add("TAPE FILE", tapeCopyLogStream.str());
}
tl.addToLog(spc);
- lc.log(log::INFO, "In RdbmsCatalogue::moveArchiveFileToRecycleBin(): ArchiveFile moved to the recycle-bin.");
+ lc.log(log::INFO, "In RdbmsCatalogue::moveArchiveFileToRecycleLog(): ArchiveFile moved to the file-recycle-log.");
} catch(exception::UserError &) {
throw;
} catch(exception::Exception &ex) {
@@ -8565,110 +8536,110 @@ void RdbmsCatalogue::moveArchiveFileToRecycleBin(const common::dataStructures::D
}
//------------------------------------------------------------------------------
-// insertArchiveFileToRecycleBin
+// copyArchiveFileToFileRecycleLog
//------------------------------------------------------------------------------
-void RdbmsCatalogue::copyArchiveFileToRecycleBin(rdbms::Conn & conn, const common::dataStructures::DeleteArchiveRequest & request) {
-
+void RdbmsCatalogue::copyArchiveFileToFileRecycleLog(rdbms::Conn & conn, const common::dataStructures::DeleteArchiveRequest & request) {
try{
- const char *const afRecycleBinInsertSql =
- "INSERT INTO ARCHIVE_FILE_RECYCLE_BIN "
- "("
- "ARCHIVE_FILE_ID,"
- "DISK_INSTANCE_NAME,"
- "DISK_FILE_ID,"
- "DISK_FILE_ID_WHEN_DELETED,"
- "DISK_FILE_UID,"
- "DISK_FILE_GID,"
- "SIZE_IN_BYTES,"
- "CHECKSUM_BLOB,"
- "CHECKSUM_ADLER32,"
- "STORAGE_CLASS_ID,"
- "CREATION_TIME,"
- "RECONCILIATION_TIME,"
- "COLLOCATION_HINT,"
- "DISK_FILE_PATH,"
- "DELETION_TIME"
- ") SELECT "
- "ARCHIVE_FILE.ARCHIVE_FILE_ID,"
- "ARCHIVE_FILE.DISK_INSTANCE_NAME,"
- "ARCHIVE_FILE.DISK_FILE_ID,"
- ":DISK_FILE_ID_WHEN_DELETED,"
- "ARCHIVE_FILE.DISK_FILE_UID,"
- "ARCHIVE_FILE.DISK_FILE_GID,"
- "ARCHIVE_FILE.SIZE_IN_BYTES,"
- "ARCHIVE_FILE.CHECKSUM_BLOB,"
- "ARCHIVE_FILE.CHECKSUM_ADLER32,"
- "ARCHIVE_FILE.STORAGE_CLASS_ID,"
- "ARCHIVE_FILE.CREATION_TIME,"
- "ARCHIVE_FILE.RECONCILIATION_TIME,"
- "ARCHIVE_FILE.COLLOCATION_HINT,"
- ":DISK_FILE_PATH,"
- ":DELETION_TIME "
- "FROM "
- "ARCHIVE_FILE "
- "JOIN "
- "STORAGE_CLASS ON ARCHIVE_FILE.STORAGE_CLASS_ID = STORAGE_CLASS.STORAGE_CLASS_ID "
- "WHERE "
- "ARCHIVE_FILE.ARCHIVE_FILE_ID = :ARCHIVE_FILE_ID";
-
- //const auto getConnTime = t.secs(utils::Timer::resetCounter);
- auto insertArchiveFileStmt = conn.createStmt(afRecycleBinInsertSql);
- insertArchiveFileStmt.bindUint64(":ARCHIVE_FILE_ID",request.archiveFileID);
- insertArchiveFileStmt.bindString(":DISK_FILE_PATH",request.diskFilePath);
- insertArchiveFileStmt.bindString(":DISK_FILE_ID_WHEN_DELETED",request.diskFileId);
- const time_t now = time(nullptr);
- insertArchiveFileStmt.bindUint64(":DELETION_TIME",now);
+ if(!request.archiveFile){
+ throw cta::exception::Exception("No archiveFile object has been set in the DeleteArchiveRequest object.");
+ }
+ const common::dataStructures::ArchiveFile & archiveFile = request.archiveFile.value();
- insertArchiveFileStmt.executeNonQuery();
+ for(auto &tapeFile: archiveFile.tapeFiles){
+ //Create one file recycle log entry per tape file
+ InsertFileRecycleLog fileRecycleLog;
+ fileRecycleLog.vid = tapeFile.vid;
+ fileRecycleLog.fSeq = tapeFile.fSeq;
+ fileRecycleLog.blockId = tapeFile.blockId;
+ fileRecycleLog.logicalSizeInBytes = tapeFile.fileSize;
+ fileRecycleLog.copyNb = tapeFile.copyNb;
+ fileRecycleLog.tapeFileCreationTime = tapeFile.creationTime;
+ fileRecycleLog.archiveFileId = archiveFile.archiveFileID;
+ fileRecycleLog.diskFilePath = request.diskFilePath;
+ fileRecycleLog.reasonLog = InsertFileRecycleLog::getDeletionReasonLog(request.requester.name,request.diskInstance);
+ fileRecycleLog.recycleLogTime = time(nullptr);
+ insertFileInFileRecycleLog(conn,fileRecycleLog);
+ }
- const char *const tfRecycleBinInsertSql =
- "INSERT INTO TAPE_FILE_RECYCLE_BIN"
- "("
+ } catch(exception::Exception &ex) {
+ ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str());
+ throw;
+ }
+}
+
+//------------------------------------------------------------------------------
+// insertFileInFileRecycleLog
+//------------------------------------------------------------------------------
+void RdbmsCatalogue::insertFileInFileRecycleLog(rdbms::Conn& conn, const InsertFileRecycleLog& fileRecycleLog){
+ try{
+ uint64_t fileRecycleLogId = getNextFileRecyleLogId(conn);
+ const char *const sql =
+ "INSERT INTO FILE_RECYCLE_LOG("
+ "FILE_RECYCLE_LOG_ID,"
"VID,"
"FSEQ,"
"BLOCK_ID,"
"LOGICAL_SIZE_IN_BYTES,"
"COPY_NB,"
- "CREATION_TIME,"
+ "TAPE_FILE_CREATION_TIME,"
"ARCHIVE_FILE_ID,"
- "SUPERSEDED_BY_VID,"
- "SUPERSEDED_BY_FSEQ,"
- "WRITE_START_WRAP,"
- "WRITE_START_LPOS,"
- "WRITE_END_WRAP,"
- "WRITE_END_LPOS,"
- "READ_START_WRAP,"
- "READ_START_LPOS,"
- "READ_END_WRAP,"
- "READ_END_LPOS"
+ "DISK_INSTANCE_NAME,"
+ "DISK_FILE_ID,"
+ "DISK_FILE_ID_WHEN_DELETED,"
+ "DISK_FILE_UID,"
+ "DISK_FILE_GID,"
+ "SIZE_IN_BYTES,"
+ "CHECKSUM_BLOB,"
+ "CHECKSUM_ADLER32,"
+ "STORAGE_CLASS_ID,"
+ "ARCHIVE_FILE_CREATION_TIME,"
+ "RECONCILIATION_TIME,"
+ "COLLOCATION_HINT,"
+ "DISK_FILE_PATH,"
+ "REASON_LOG,"
+ "RECYCLE_LOG_TIME"
") SELECT "
- "TAPE_FILE.VID,"
- "TAPE_FILE.FSEQ,"
- "TAPE_FILE.BLOCK_ID,"
- "TAPE_FILE.LOGICAL_SIZE_IN_BYTES,"
- "TAPE_FILE.COPY_NB,"
- "TAPE_FILE.CREATION_TIME,"
- "TAPE_FILE.ARCHIVE_FILE_ID,"
- "TAPE_FILE.SUPERSEDED_BY_VID,"
- "TAPE_FILE.SUPERSEDED_BY_FSEQ,"
- "TAPE_FILE.WRITE_START_WRAP,"
- "TAPE_FILE.WRITE_START_LPOS,"
- "TAPE_FILE.WRITE_END_WRAP,"
- "TAPE_FILE.WRITE_END_LPOS,"
- "TAPE_FILE.READ_START_WRAP,"
- "TAPE_FILE.READ_START_LPOS,"
- "TAPE_FILE.READ_END_WRAP,"
- "TAPE_FILE.READ_END_LPOS "
+ ":FILE_RECYCLE_LOG_ID,"
+ ":VID,"
+ ":FSEQ,"
+ ":BLOCK_ID,"
+ ":LOGICAL_SIZE_IN_BYTES,"
+ ":COPY_NB,"
+ ":TAPE_FILE_CREATION_TIME,"
+ ":ARCHIVE_FILE_ID,"
+ "ARCHIVE_FILE.DISK_INSTANCE_NAME AS DISK_INSTANCE_NAME,"
+ "ARCHIVE_FILE.DISK_FILE_ID AS DISK_FILE_ID,"
+ "ARCHIVE_FILE.DISK_FILE_ID AS DISK_FILE_ID_2,"
+ "ARCHIVE_FILE.DISK_FILE_UID AS DISK_FILE_UID,"
+ "ARCHIVE_FILE.DISK_FILE_GID AS DISK_FILE_GID,"
+ "ARCHIVE_FILE.SIZE_IN_BYTES AS SIZE_IN_BYTES,"
+ "ARCHIVE_FILE.CHECKSUM_BLOB AS CHECKSUM_BLOB,"
+ "ARCHIVE_FILE.CHECKSUM_ADLER32 AS CHECKSUM_ADLER32,"
+ "ARCHIVE_FILE.STORAGE_CLASS_ID AS STORAGE_CLASS_ID,"
+ "ARCHIVE_FILE.CREATION_TIME AS ARCHIVE_FILE_CREATION_TIME,"
+ "ARCHIVE_FILE.RECONCILIATION_TIME AS RECONCILIATION_TIME,"
+ "ARCHIVE_FILE.COLLOCATION_HINT AS COLLOCATION_HINT,"
+ ":DISK_FILE_PATH,"
+ ":REASON_LOG,"
+ ":RECYCLE_LOG_TIME "
"FROM "
- "TAPE_FILE "
- "WHERE TAPE_FILE.ARCHIVE_FILE_ID = :ARCHIVE_FILE_ID";
-
- auto insertTapeFileStmt = conn.createStmt(tfRecycleBinInsertSql);
- insertTapeFileStmt.bindUint64(":ARCHIVE_FILE_ID",request.archiveFileID);
-
- insertTapeFileStmt.executeNonQuery();
- } catch(exception::UserError &) {
- throw;
+ "ARCHIVE_FILE "
+ "WHERE "
+ "ARCHIVE_FILE.ARCHIVE_FILE_ID = :ARCHIVE_FILE_ID_2";
+ auto stmt = conn.createStmt(sql);
+ stmt.bindUint64(":FILE_RECYCLE_LOG_ID",fileRecycleLogId);
+ stmt.bindString(":VID",fileRecycleLog.vid);
+ stmt.bindUint64(":FSEQ",fileRecycleLog.fSeq);
+ stmt.bindUint64(":BLOCK_ID",fileRecycleLog.blockId);
+ stmt.bindUint64(":LOGICAL_SIZE_IN_BYTES",fileRecycleLog.logicalSizeInBytes);
+ stmt.bindUint8(":COPY_NB",fileRecycleLog.copyNb);
+ stmt.bindUint64(":TAPE_FILE_CREATION_TIME",fileRecycleLog.tapeFileCreationTime);
+ stmt.bindString(":DISK_FILE_PATH",fileRecycleLog.diskFilePath);
+ stmt.bindUint64(":ARCHIVE_FILE_ID",fileRecycleLog.archiveFileId);
+ stmt.bindString(":REASON_LOG",fileRecycleLog.reasonLog);
+ stmt.bindUint64(":RECYCLE_LOG_TIME",fileRecycleLog.recycleLogTime);
+ stmt.bindUint64(":ARCHIVE_FILE_ID_2",fileRecycleLog.archiveFileId);
+ stmt.executeNonQuery();
} catch(exception::Exception &ex) {
ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str());
throw;
@@ -8805,7 +8776,7 @@ std::list<InsertFileRecycleLog> RdbmsCatalogue::insertOldCopiesOfFilesIfAnyOnFil
while(rset.next()){
cta::catalogue::InsertFileRecycleLog fileRecycleLog;
fileRecycleLog.vid = rset.columnString("VID");
- fileRecycleLog.fseq = rset.columnUint64("FSEQ");
+ fileRecycleLog.fSeq = rset.columnUint64("FSEQ");
fileRecycleLog.blockId = rset.columnUint64("BLOCK_ID");
fileRecycleLog.logicalSizeInBytes = rset.columnUint64("LOGICAL_SIZE_IN_BYTES");
fileRecycleLog.copyNb = rset.columnUint8("COPY_NB");
@@ -8818,71 +8789,7 @@ std::list<InsertFileRecycleLog> RdbmsCatalogue::insertOldCopiesOfFilesIfAnyOnFil
}
{
for(auto & fileRecycleLog: fileRecycleLogsToInsert){
- uint64_t fileRecycleLogId = getNextFileRecyleLogId(conn);
- const char *const sql =
- "INSERT INTO FILE_RECYCLE_LOG("
- "FILE_RECYCLE_LOG_ID,"
- "VID,"
- "FSEQ,"
- "BLOCK_ID,"
- "LOGICAL_SIZE_IN_BYTES,"
- "COPY_NB,"
- "TAPE_FILE_CREATION_TIME,"
- "ARCHIVE_FILE_ID,"
- "DISK_INSTANCE_NAME,"
- "DISK_FILE_ID,"
- "DISK_FILE_ID_WHEN_DELETED,"
- "DISK_FILE_UID,"
- "DISK_FILE_GID,"
- "SIZE_IN_BYTES,"
- "CHECKSUM_BLOB,"
- "CHECKSUM_ADLER32,"
- "STORAGE_CLASS_ID,"
- "ARCHIVE_FILE_CREATION_TIME,"
- "RECONCILIATION_TIME,"
- "COLLOCATION_HINT,"
- "REASON_LOG,"
- "RECYCLE_LOG_TIME"
- ") SELECT "
- ":FILE_RECYCLE_LOG_ID,"
- ":VID,"
- ":FSEQ,"
- ":BLOCK_ID,"
- ":LOGICAL_SIZE_IN_BYTES,"
- ":COPY_NB,"
- ":TAPE_FILE_CREATION_TIME,"
- ":ARCHIVE_FILE_ID,"
- "ARCHIVE_FILE.DISK_INSTANCE_NAME AS DISK_INSTANCE_NAME,"
- "ARCHIVE_FILE.DISK_FILE_ID AS DISK_FILE_ID,"
- "ARCHIVE_FILE.DISK_FILE_ID AS DISK_FILE_ID_2,"
- "ARCHIVE_FILE.DISK_FILE_UID AS DISK_FILE_UID,"
- "ARCHIVE_FILE.DISK_FILE_GID AS DISK_FILE_GID,"
- "ARCHIVE_FILE.SIZE_IN_BYTES AS SIZE_IN_BYTES,"
- "ARCHIVE_FILE.CHECKSUM_BLOB AS CHECKSUM_BLOB,"
- "ARCHIVE_FILE.CHECKSUM_ADLER32 AS CHECKSUM_ADLER32,"
- "ARCHIVE_FILE.STORAGE_CLASS_ID AS STORAGE_CLASS_ID,"
- "ARCHIVE_FILE.CREATION_TIME AS ARCHIVE_FILE_CREATION_TIME,"
- "ARCHIVE_FILE.RECONCILIATION_TIME AS RECONCILIATION_TIME,"
- "ARCHIVE_FILE.COLLOCATION_HINT AS COLLOCATION_HINT,"
- ":REASON_LOG,"
- ":RECYCLE_LOG_TIME "
- "FROM "
- "ARCHIVE_FILE "
- "WHERE "
- "ARCHIVE_FILE.ARCHIVE_FILE_ID = :ARCHIVE_FILE_ID_2";
- auto stmt = conn.createStmt(sql);
- stmt.bindUint64(":FILE_RECYCLE_LOG_ID",fileRecycleLogId);
- stmt.bindString(":VID",fileRecycleLog.vid);
- stmt.bindUint64(":FSEQ",fileRecycleLog.fseq);
- stmt.bindUint64(":BLOCK_ID",fileRecycleLog.blockId);
- stmt.bindUint64(":LOGICAL_SIZE_IN_BYTES",fileRecycleLog.logicalSizeInBytes);
- stmt.bindUint8(":COPY_NB",fileRecycleLog.copyNb);
- stmt.bindUint64(":TAPE_FILE_CREATION_TIME",fileRecycleLog.tapeFileCreationTime);
- stmt.bindUint64(":ARCHIVE_FILE_ID",fileRecycleLog.archiveFileId);
- stmt.bindString(":REASON_LOG",fileRecycleLog.reasonLog);
- stmt.bindUint64(":RECYCLE_LOG_TIME",fileRecycleLog.recycleLogTime);
- stmt.bindUint64(":ARCHIVE_FILE_ID_2",fileRecycleLog.archiveFileId);
- stmt.executeNonQuery();
+ insertFileInFileRecycleLog(conn,fileRecycleLog);
}
}
} catch(exception::Exception &ex) {
@@ -8928,6 +8835,19 @@ void RdbmsCatalogue::deleteFilesFromRecycleBin(rdbms::Conn & conn, const std::st
// deleteFilesFromRecycleLog
//------------------------------------------------------------------------------
void RdbmsCatalogue::deleteFilesFromRecycleLog(const std::string& vid, log::LogContext& lc) {
+ try {
+ auto conn = m_connPool.getConn();
+ deleteFilesFromRecycleLog(conn,vid,lc);
+ } catch(exception::Exception &ex) {
+ ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str());
+ throw;
+ }
+}
+
+//------------------------------------------------------------------------------
+// deleteFilesFromRecycleLog
+//------------------------------------------------------------------------------
+void RdbmsCatalogue::deleteFilesFromRecycleLog(rdbms::Conn & conn, const std::string& vid, log::LogContext& lc) {
try {
const char *const deleteFilesFromRecycleLogSql =
"DELETE FROM "
@@ -8937,7 +8857,6 @@ void RdbmsCatalogue::deleteFilesFromRecycleLog(const std::string& vid, log::LogC
cta::utils::Timer t;
log::TimingList tl;
- auto conn = m_connPool.getConn();
auto selectFileStmt = conn.createStmt(deleteFilesFromRecycleLogSql);
selectFileStmt.bindString(":VID",vid);
selectFileStmt.executeNonQuery();
diff --git a/catalogue/RdbmsCatalogue.hpp b/catalogue/RdbmsCatalogue.hpp
index 828e2991af..6368174aa2 100644
--- a/catalogue/RdbmsCatalogue.hpp
+++ b/catalogue/RdbmsCatalogue.hpp
@@ -537,14 +537,6 @@ public:
*/
void checkTapeForLabel(const std::string &vid) override;
- /**
- * Returns the number of non superseded files contained in the tape identified by its vid
- * @param conn the database connection
- * @param vid the vid in which we will count non superseded files
- * @return the number of non superseded files on the vid
- */
- uint64_t getNbNonSupersededFilesOnTape(rdbms::Conn &conn, const std::string &vid) const;
-
/**
* Returns the number of any files contained in the tape identified by its vid
* @param vid the vid in which we will count non superseded files
@@ -1854,29 +1846,37 @@ protected:
const std::string &diskFileId) override;
/**
- * Insert the tape files and ArchiveFiles entries in the recycle-bin and delete
- * them from the TAPE_FILE and ARCHIVE_FILE tables
+ * Insert the ArchiveFile and all its tape files in the FILE_RECYCLE_LOG table.
+ * There will be one entry on the FILE_RECYCLE_LOG table per deleted tape file
+ *
* @param request the DeleteRequest object that holds information about the file to delete.
* @param lc the logContext
*/
- void moveArchiveFileToRecycleBin(const common::dataStructures::DeleteArchiveRequest &request,
+ void moveArchiveFileToRecycleLog(const common::dataStructures::DeleteArchiveRequest &request,
log::LogContext & lc) override;
/**
- * Copy the archiveFile and the associated tape files from the ARCHIVE_FILE and TAPE_FILE tables to the recycle-bin tables
+ * Copy the archiveFile and the associated tape files from the ARCHIVE_FILE and TAPE_FILE tables to the FILE_RECYCLE_LOG table
* and deletes the ARCHIVE_FILE and TAPE_FILE entries.
* @param conn the database connection
- * @param request the request that contains the necessary informations to identify the archiveFile to copy to the recycle-bin
+ * @param request the request that contains the necessary informations to identify the archiveFile to copy to the FILE_RECYCLE_LOG table
* @param lc the log context
*/
- virtual void copyArchiveFileToRecycleBinAndDelete(rdbms::Conn & conn,const common::dataStructures::DeleteArchiveRequest &request, log::LogContext & lc) = 0;
+ virtual void copyArchiveFileToFileRecyleLogAndDelete(rdbms::Conn & conn,const common::dataStructures::DeleteArchiveRequest &request, log::LogContext & lc) = 0;
/**
* Copies the ARCHIVE_FILE and TAPE_FILE entries to the recycle-bin tables
* @param conn the database connection
* @param request the request that contains the necessary informations to identify the archiveFile to copy to the recycle-bin
*/
- void copyArchiveFileToRecycleBin(rdbms::Conn & conn, const common::dataStructures::DeleteArchiveRequest & request);
+ void copyArchiveFileToFileRecycleLog(rdbms::Conn & conn, const common::dataStructures::DeleteArchiveRequest & request);
+
+ /**
+ * Insert the file in the FILE_RECYCLE_LOG table
+ * @param conn the database connection
+ * @param fileRecycleLog the file to insert into the FILE_RECYCLE_LOG table
+ */
+ void insertFileToRecycleLog(rdbms::Conn & conn, const InsertFileRecycleLog & fileRecycleLog);
/**
* Deletes the ArchiveFile from the ARCHIVE_FILE table
@@ -1926,6 +1926,19 @@ protected:
*/
void deleteFilesFromRecycleLog(const std::string & vid, log::LogContext & lc);
+ /**
+ * Deletes all the log entries corresponding to the vid passed in parameter.
+ *
+ * Please note that this method is idempotent. If there are no recycle log
+ * entries associated to the vid passed in parameter, the method will return
+ * without any error.
+ *
+ *@param conn, the database connection
+ * @param vid, the vid of the files to be deleted
+ * @param lc, the logContext
+ */
+ void deleteFilesFromRecycleLog(rdbms::Conn & conn, const std::string & vid, log::LogContext & lc);
+
/**
* Delete the TapeFiles and the ArchiveFile from the recycle-bin in one transaction
* @param conn the database connection
@@ -1959,6 +1972,13 @@ protected:
*/
std::list<cta::catalogue::InsertFileRecycleLog> insertOldCopiesOfFilesIfAnyOnFileRecycleLog(rdbms::Conn & conn,const common::dataStructures::TapeFile &tapeFile, const uint64_t archiveFileId);
+ /**
+ * Insert the file passed in parameter in the FILE_RECYCLE_LOG table
+ * @param conn the database connection
+ * @param fileRecycleLog the file to insert on the FILE_RECYCLE_LOG table
+ */
+ void insertFileInFileRecycleLog(rdbms::Conn & conn, const InsertFileRecycleLog & fileRecycleLog);
+
/**
* Generates the SELECT statement required to search for tapes using 100 tape
* VIDs. Each tape VID is represented in the SQL by a bind parameter with the
diff --git a/catalogue/SqliteCatalogue.cpp b/catalogue/SqliteCatalogue.cpp
index eee30e6fa0..365232ffa8 100644
--- a/catalogue/SqliteCatalogue.cpp
+++ b/catalogue/SqliteCatalogue.cpp
@@ -592,14 +592,14 @@ void SqliteCatalogue::fileWrittenToTape(rdbms::Conn &conn, const TapeFileWritten
//------------------------------------------------------------------------------
// copyArchiveFileToRecycleBinAndDelete
//------------------------------------------------------------------------------
-void SqliteCatalogue::copyArchiveFileToRecycleBinAndDelete(rdbms::Conn & conn, const common::dataStructures::DeleteArchiveRequest &request, log::LogContext & lc) {
+void SqliteCatalogue::copyArchiveFileToFileRecyleLogAndDelete(rdbms::Conn & conn, const common::dataStructures::DeleteArchiveRequest &request, log::LogContext & lc) {
try {
utils::Timer t;
log::TimingList tl;
//We currently do an INSERT INTO and a DELETE FROM
//in a single transaction
conn.executeNonQuery("BEGIN TRANSACTION");
- copyArchiveFileToRecycleBin(conn,request);
+ copyArchiveFileToFileRecycleLog(conn,request);
tl.insertAndReset("insertToRecycleBinTime",t);
setTapeDirty(conn,request.archiveFileID);
tl.insertAndReset("setTapeDirtyTime",t);
diff --git a/catalogue/SqliteCatalogue.hpp b/catalogue/SqliteCatalogue.hpp
index 85ed56ed78..473bc542c8 100644
--- a/catalogue/SqliteCatalogue.hpp
+++ b/catalogue/SqliteCatalogue.hpp
@@ -202,13 +202,13 @@ protected:
void filesWrittenToTape(const std::set<TapeItemWrittenPointer> &events) override;
/**
- * Copy the archiveFile and the associated tape files from the ARCHIVE_FILE and TAPE_FILE tables to the recycle-bin tables
+ * Copy the archiveFile and the associated tape files from the ARCHIVE_FILE and TAPE_FILE tables to the FILE_RECYCLE_LOG table
* and deletes the ARCHIVE_FILE and TAPE_FILE entries.
* @param conn the database connection
- * @param request the request that contains the necessary informations to identify the archiveFile to copy to the recycle-bin
+ * @param request the request that contains the necessary informations to identify the archiveFile to copy to the FILE_RECYCLE_LOG table
* @param lc the log context
*/
- void copyArchiveFileToRecycleBinAndDelete(rdbms::Conn & conn, const common::dataStructures::DeleteArchiveRequest &request, log::LogContext & lc) override;
+ void copyArchiveFileToFileRecyleLogAndDelete(rdbms::Conn & conn, const common::dataStructures::DeleteArchiveRequest &request, log::LogContext & lc) override;
/**
* Delete the TapeFiles and the ArchiveFile from the recycle-bin in one transaction
diff --git a/common/dataStructures/DeleteArchiveRequest.hpp b/common/dataStructures/DeleteArchiveRequest.hpp
index ec7c75bab2..7735227f52 100644
--- a/common/dataStructures/DeleteArchiveRequest.hpp
+++ b/common/dataStructures/DeleteArchiveRequest.hpp
@@ -25,6 +25,7 @@
#include "common/optional.hpp"
#include "common/dataStructures/RequesterIdentity.hpp"
+#include "ArchiveFile.hpp"
namespace cta {
namespace common {
@@ -49,6 +50,8 @@ struct DeleteArchiveRequest {
std::string diskFileId;
std::string diskInstance;
time_t recycleTime;
+ //In the case the ArchiveFile does not exist yet, it will not be set
+ cta::optional<ArchiveFile> archiveFile;
}; // struct DeleteArchiveRequest
diff --git a/scheduler/Scheduler.cpp b/scheduler/Scheduler.cpp
index 9493863153..a1ea61bc09 100644
--- a/scheduler/Scheduler.cpp
+++ b/scheduler/Scheduler.cpp
@@ -272,7 +272,7 @@ void Scheduler::deleteArchive(const std::string &instanceName, const common::dat
m_db.cancelArchive(request,lc);
}
tl.insertAndReset("schedulerDbTime",t);
- m_catalogue.moveArchiveFileToRecycleBin(request,lc);
+ m_catalogue.moveArchiveFileToRecycleLog(request,lc);
tl.insertAndReset("catalogueTime",t);
log::ScopedParamContainer spc(lc);
tl.addToLog(spc);
diff --git a/xroot_plugins/XrdSsiCtaRequestMessage.cpp b/xroot_plugins/XrdSsiCtaRequestMessage.cpp
index 758076dbfd..60bba26ed7 100644
--- a/xroot_plugins/XrdSsiCtaRequestMessage.cpp
+++ b/xroot_plugins/XrdSsiCtaRequestMessage.cpp
@@ -709,14 +709,23 @@ void RequestMessage::processDELETE(const cta::eos::Notification ¬ification, c
// Delete the file from the catalogue or from the objectstore if archive request is created
cta::utils::Timer t;
+ cta::log::TimingList tl;
+ try {
+ request.archiveFile = m_catalogue.getArchiveFileById(request.archiveFileID);
+ tl.insertAndReset("catalogueGetArchiveFileByIdTime",t);
+ } catch (cta::exception::Exception &ex){
+ log::ScopedParamContainer spc(m_lc);
+ spc.add("fileId", request.archiveFileID);
+ m_lc.log(log::WARNING, "Ignoring request to delete archive file because it does not exist in the catalogue");
+ }
m_scheduler.deleteArchive(m_cliIdentity.username, request, m_lc);
-
+ tl.insertAndReset("schedulerTime",t);
// Create a log entry
cta::log::ScopedParamContainer params(m_lc);
params.add("fileId", request.archiveFileID)
.add("address", (request.address ? request.address.value() : "null"))
- .add("filePath",request.diskFilePath)
- .add("schedulerTime", t.secs());
+ .add("filePath",request.diskFilePath);
+ tl.addToLog(params);
m_lc.log(cta::log::INFO, "In RequestMessage::processDELETE(): archive file deleted.");
// Set response type
--
GitLab