Skip to content

GitLab

Commit cae7b773 authored by Cedric Caffy's avatar Cedric Caffy
Browse files

[catalogue] Changed the reclaim logic + file deletion logic

parent 5df50912
Show whitespace changes
Inline Side-by-side
catalogue/Catalogue.hpp
View file @ cae7b773
......@@ -1024,12 +1024,13 @@ public:
const std::string &diskFileId) = 0;
/**
* Insert the tape files and ArchiveFiles entries in the recycle-bin and delete
* them from the TAPE_FILE and ARCHIVE_FILE tables
* Insert the ArchiveFile and all its tape files in the FILE_RECYCLE_LOG table.
* There will be one entry on the FILE_RECYCLE_LOG table per deleted tape file
*
* @param request the DeleteRequest object that holds information about the file to delete.
* @param lc the logContext
*/
virtual void moveArchiveFileToRecycleBin(const common::dataStructures::DeleteArchiveRequest &request,
virtual void moveArchiveFileToRecycleLog(const common::dataStructures::DeleteArchiveRequest &request,
log::LogContext & lc) = 0;
/**
......
catalogue/CatalogueRetryWrapper.hpp
View file @ cae7b773
......@@ -598,9 +598,9 @@ public:
return retryOnLostConnection(m_log, [&]{return m_catalogue->updateDiskFileId(archiveFileId, diskInstance, diskFileId);}, m_maxTriesToConnect);
}
void moveArchiveFileToRecycleBin(const common::dataStructures::DeleteArchiveRequest &request,
void moveArchiveFileToRecycleLog(const common::dataStructures::DeleteArchiveRequest &request,
log::LogContext & lc) override {
return retryOnLostConnection(m_log,[&]{return m_catalogue->moveArchiveFileToRecycleBin(request,lc);},m_maxTriesToConnect);
return retryOnLostConnection(m_log,[&]{return m_catalogue->moveArchiveFileToRecycleLog(request,lc);},m_maxTriesToConnect);
}
protected:
......
catalogue/CatalogueTest.cpp
View file @ cae7b773
......@@ -9169,6 +9169,8 @@ TEST_P(cta_catalogue_CatalogueTest, prepareToRetrieveFileUsingArchiveFileId_retu
file2Written.tapeDrive = tapeDrive;
m_catalogue->filesWrittenToTape(file2WrittenSet);
ASSERT_TRUE(m_catalogue->getFileRecycleLogItor().hasMore());
const std::string mountPolicyName = "mount_policy";
const uint64_t archivePriority = 1;
const uint64_t minArchiveRequestAge = 2;
......@@ -15721,13 +15723,14 @@ TEST_P(cta_catalogue_CatalogueTest, moveFilesToRecycleBin) {
req.diskFileId = tapeItem->diskFileId;
req.diskFilePath = tapeItem->diskFilePath;
req.diskInstance = tapeItem->diskInstance;
ASSERT_NO_THROW(m_catalogue->moveArchiveFileToRecycleBin(req,dummyLc));
req.archiveFile = m_catalogue->getArchiveFileById(tapeItem->archiveFileId);
ASSERT_NO_THROW(m_catalogue->moveArchiveFileToRecycleLog(req,dummyLc));
}
ASSERT_FALSE(m_catalogue->getArchiveFilesItor().hasMore());
std::vector<common::dataStructures::DeletedArchiveFile> deletedArchiveFiles;
std::vector<common::dataStructures::FileRecycleLog> deletedArchiveFiles;
{
auto itor = m_catalogue->getDeletedArchiveFilesItor();
auto itor = m_catalogue->getFileRecycleLogItor();
while(itor.hasMore()){
deletedArchiveFiles.push_back(itor.next());
}
......@@ -15747,40 +15750,36 @@ TEST_P(cta_catalogue_CatalogueTest, moveFilesToRecycleBin) {
std::ostringstream diskFilePath;
diskFilePath << "/test/file"<<i;
ASSERT_EQ(i,deletedArchiveFile.archiveFileID);
ASSERT_EQ(diskInstance,deletedArchiveFile.diskInstance);
ASSERT_EQ(i,deletedArchiveFile.archiveFileId);
ASSERT_EQ(diskInstance,deletedArchiveFile.diskInstanceName);
ASSERT_EQ(diskFileId.str(),deletedArchiveFile.diskFileId);
ASSERT_EQ(diskFilePath.str(),deletedArchiveFile.diskFilePath);
ASSERT_EQ(PUBLIC_DISK_USER,deletedArchiveFile.diskFileInfo.owner_uid);
ASSERT_EQ(PUBLIC_DISK_GROUP,deletedArchiveFile.diskFileInfo.gid);
ASSERT_EQ(archiveFileSize,deletedArchiveFile.fileSize);
ASSERT_EQ(PUBLIC_DISK_USER,deletedArchiveFile.diskFileUid);
ASSERT_EQ(PUBLIC_DISK_GROUP,deletedArchiveFile.diskFileGid);
ASSERT_EQ(archiveFileSize,deletedArchiveFile.sizeInBytes);
ASSERT_EQ(cta::checksum::ChecksumBlob(checksum::ADLER32, "1357"),deletedArchiveFile.checksumBlob);
ASSERT_EQ(m_storageClassSingleCopy.name, deletedArchiveFile.storageClass);
ASSERT_EQ(m_storageClassSingleCopy.name, deletedArchiveFile.storageClassName);
ASSERT_EQ(diskFileId.str(),deletedArchiveFile.diskFileIdWhenDeleted);
auto tapeFile = deletedArchiveFile.tapeFiles.at(1);
ASSERT_EQ(tape1.vid, tapeFile.vid);
ASSERT_EQ(i,tapeFile.fSeq);
ASSERT_EQ(i * 100,tapeFile.blockId);
ASSERT_EQ(1, tapeFile.copyNb);
ASSERT_EQ(archiveFileSize,tapeFile.fileSize);
ASSERT_EQ(tape1.vid, deletedArchiveFile.vid);
ASSERT_EQ(i,deletedArchiveFile.fSeq);
ASSERT_EQ(i * 100,deletedArchiveFile.blockId);
ASSERT_EQ(1, deletedArchiveFile.copyNb);
ASSERT_EQ(archiveFileSize,deletedArchiveFile.logicalSizeInBytes);
}
//Let's try the deletion of the files from the recycle-bin.
for(uint64_t i = 1; i <= nbArchiveFiles; i++) {
m_catalogue->deleteFileFromRecycleBin(i,dummyLc);
m_catalogue->deleteFilesFromRecycleLog(tape1.vid,dummyLc);
}
{
auto itor = m_catalogue->getDeletedArchiveFilesItor();
auto itor = m_catalogue->getFileRecycleLogItor();
ASSERT_FALSE(itor.hasMore());
}
//Delete an archive file from the recycle-bin should be idempotent
ASSERT_NO_THROW(m_catalogue->deleteFileFromRecycleBin(12532,dummyLc));
}
 
TEST_P(cta_catalogue_CatalogueTest, reclaimTapeRemovesFilesFromRecycleBin) {
TEST_P(cta_catalogue_CatalogueTest, reclaimTapeRemovesFilesFromRecycleLog) {
using namespace cta;
 
const bool logicalLibraryIsDisabled= false;
......@@ -15851,27 +15850,27 @@ TEST_P(cta_catalogue_CatalogueTest, reclaimTapeRemovesFilesFromRecycleBin) {
req.diskFileId = tapeItem->diskFileId;
req.diskFilePath = tapeItem->diskFilePath;
req.diskInstance = tapeItem->diskInstance;
ASSERT_NO_THROW(m_catalogue->moveArchiveFileToRecycleBin(req,dummyLc));
req.archiveFile = m_catalogue->getArchiveFileById(tapeItem->archiveFileId);
ASSERT_NO_THROW(m_catalogue->moveArchiveFileToRecycleLog(req,dummyLc));
}
ASSERT_FALSE(m_catalogue->getArchiveFilesItor().hasMore());
std::vector<common::dataStructures::DeletedArchiveFile> deletedArchiveFiles;
std::vector<common::dataStructures::FileRecycleLog> deletedArchiveFiles;
{
auto itor = m_catalogue->getDeletedArchiveFilesItor();
auto itor = m_catalogue->getFileRecycleLogItor();
while(itor.hasMore()){
deletedArchiveFiles.push_back(itor.next());
}
}
//And test that these files are there.
//Run the unit test for all the databases
//And test that these files are in the recycle log
ASSERT_EQ(nbArchiveFiles,deletedArchiveFiles.size());
ASSERT_TRUE(m_catalogue->getFileRecycleLogItor().hasMore());
//Reclaim the tape
m_catalogue->setTapeFull(m_admin, tape1.vid, true);
m_catalogue->reclaimTape(m_admin, tape1.vid, dummyLc);
{
auto itor = m_catalogue->getDeletedArchiveFilesItor();
auto itor = m_catalogue->getFileRecycleLogItor();
ASSERT_FALSE(itor.hasMore());
}
}
......
catalogue/DummyCatalogue.hpp
View file @ cae7b773
......@@ -164,7 +164,7 @@ public:
void tapeMountedForRetrieve(const std::string& vid, const std::string& drive) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
bool tapePoolExists(const std::string& tapePoolName) const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
void updateDiskFileId(uint64_t archiveFileId, const std::string &diskInstance, const std::string &diskFileId) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
void moveArchiveFileToRecycleBin(const common::dataStructures::DeleteArchiveRequest &request,
void moveArchiveFileToRecycleLog(const common::dataStructures::DeleteArchiveRequest &request,
log::LogContext & lc) override {throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented");}
// Special functions for unit tests.
void addEnabledTape(const std::string & vid) {
......
catalogue/InsertFileRecycleLog.hpp
View file @ cae7b773
......@@ -28,19 +28,23 @@ namespace cta { namespace catalogue {
class InsertFileRecycleLog {
public:
std::string vid;
uint64_t fseq;
uint64_t fSeq;
uint64_t blockId;
uint64_t logicalSizeInBytes;
uint8_t copyNb;
time_t tapeFileCreationTime;
uint64_t archiveFileId;
std::string diskFilePath;
cta::optional<std::string> diskFilePath;
std::string reasonLog;
time_t recycleLogTime;
static std::string getRepackReasonLog(){
return "REPACK";
}
static std::string getDeletionReasonLog(const std::string & deleterName, const std::string & diskInstanceName){
return "File deleted by " + deleterName + "from the " + diskInstanceName + "instance";
}
};
}}
......
catalogue/MysqlCatalogue.cpp
View file @ cae7b773
......@@ -729,14 +729,14 @@ void MysqlCatalogue::DO_NOT_USE_deleteArchiveFile_DO_NOT_USE(const std::string &
//------------------------------------------------------------------------------
// copyArchiveFileToRecycleBinAndDelete
//------------------------------------------------------------------------------
void MysqlCatalogue::copyArchiveFileToRecycleBinAndDelete(rdbms::Conn & conn, const common::dataStructures::DeleteArchiveRequest &request, log::LogContext & lc){
void MysqlCatalogue::copyArchiveFileToFileRecyleLogAndDelete(rdbms::Conn & conn, const common::dataStructures::DeleteArchiveRequest &request, log::LogContext & lc){
try {
utils::Timer t;
log::TimingList tl;
//We currently do an INSERT INTO and a DELETE FROM
//in a single transaction
conn.executeNonQuery("START TRANSACTION");
copyArchiveFileToRecycleBin(conn,request);
copyArchiveFileToFileRecycleLog(conn,request);
tl.insertAndReset("insertToRecycleBinTime",t);
setTapeDirty(conn,request.archiveFileID);
tl.insertAndReset("setTapeDirtyTime",t);
......
catalogue/MysqlCatalogue.hpp
View file @ cae7b773
......@@ -194,13 +194,13 @@ protected:
log::LogContext &lc) override;
/**
* Copy the archiveFile and the associated tape files from the ARCHIVE_FILE and TAPE_FILE tables to the recycle-bin tables
* Copy the archiveFile and the associated tape files from the ARCHIVE_FILE and TAPE_FILE tables to the FILE_RECYCLE_LOG table
* and deletes the ARCHIVE_FILE and TAPE_FILE entries.
* @param conn the database connection
* @param request the request that contains the necessary informations to identify the archiveFile to copy to the recycle-bin
* @param request the request that contains the necessary informations to identify the archiveFile to copy to the FILE_RECYCLE_LOG table
* @param lc the log context
*/
void copyArchiveFileToRecycleBinAndDelete(rdbms::Conn & conn,const common::dataStructures::DeleteArchiveRequest &request, log::LogContext & lc) override;
void copyArchiveFileToFileRecyleLogAndDelete(rdbms::Conn & conn,const common::dataStructures::DeleteArchiveRequest &request, log::LogContext & lc) override;
/**
* Delete the TapeFiles and the ArchiveFile from the recycle-bin in one transaction
......
catalogue/OracleCatalogue.cpp
View file @ cae7b773
......@@ -593,7 +593,7 @@ void OracleCatalogue::filesWrittenToTape(const std::set<TapeItemWrittenPointer>
auto stmt = conn.createStmt(sql);
stmt.bindString(":VID",recycledFile.vid);
stmt.bindUint64(":FSEQ",recycledFile.fseq);
stmt.bindUint64(":FSEQ",recycledFile.fSeq);
stmt.executeNonQuery();
}
......@@ -788,7 +788,7 @@ std::list<cta::catalogue::InsertFileRecycleLog> OracleCatalogue::insertOldCopies
while(rset.next()){
cta::catalogue::InsertFileRecycleLog fileRecycleLog;
fileRecycleLog.vid = rset.columnString("VID");
fileRecycleLog.fseq = rset.columnUint64("FSEQ");
fileRecycleLog.fSeq = rset.columnUint64("FSEQ");
fileRecycleLog.blockId = rset.columnUint64("BLOCK_ID");
fileRecycleLog.logicalSizeInBytes = rset.columnUint64("LOGICAL_SIZE_IN_BYTES");
fileRecycleLog.copyNb = rset.columnUint8("COPY_NB");
......@@ -801,70 +801,7 @@ std::list<cta::catalogue::InsertFileRecycleLog> OracleCatalogue::insertOldCopies
}
{
for(auto & fileRecycleLog: fileRecycleLogsToInsert){
const char *const sql =
"INSERT INTO FILE_RECYCLE_LOG("
"FILE_RECYCLE_LOG_ID,"
"VID,"
"FSEQ,"
"BLOCK_ID,"
"LOGICAL_SIZE_IN_BYTES,"
"COPY_NB,"
"TAPE_FILE_CREATION_TIME,"
"ARCHIVE_FILE_ID,"
"DISK_INSTANCE_NAME,"
"DISK_FILE_ID,"
"DISK_FILE_ID_WHEN_DELETED,"
"DISK_FILE_UID,"
"DISK_FILE_GID,"
"SIZE_IN_BYTES,"
"CHECKSUM_BLOB,"
"CHECKSUM_ADLER32,"
"STORAGE_CLASS_ID,"
"ARCHIVE_FILE_CREATION_TIME,"
"RECONCILIATION_TIME,"
"COLLOCATION_HINT,"
"REASON_LOG,"
"RECYCLE_LOG_TIME"
") SELECT "
"FILE_RECYCLE_LOG_ID_SEQ.NEXTVAL,"
":VID,"
":FSEQ,"
":BLOCK_ID,"
":LOGICAL_SIZE_IN_BYTES,"
":COPY_NB,"
":TAPE_FILE_CREATION_TIME,"
":ARCHIVE_FILE_ID,"
"ARCHIVE_FILE.DISK_INSTANCE_NAME AS DISK_INSTANCE_NAME,"
"ARCHIVE_FILE.DISK_FILE_ID AS DISK_FILE_ID,"
"ARCHIVE_FILE.DISK_FILE_ID AS DISK_FILE_ID_2,"
"ARCHIVE_FILE.DISK_FILE_UID AS DISK_FILE_UID,"
"ARCHIVE_FILE.DISK_FILE_GID AS DISK_FILE_GID,"
"ARCHIVE_FILE.SIZE_IN_BYTES AS SIZE_IN_BYTES,"
"ARCHIVE_FILE.CHECKSUM_BLOB AS CHECKSUM_BLOB,"
"ARCHIVE_FILE.CHECKSUM_ADLER32 AS CHECKSUM_ADLER32,"
"ARCHIVE_FILE.STORAGE_CLASS_ID AS STORAGE_CLASS_ID,"
"ARCHIVE_FILE.CREATION_TIME AS ARCHIVE_FILE_CREATION_TIME,"
"ARCHIVE_FILE.RECONCILIATION_TIME AS RECONCILIATION_TIME,"
"ARCHIVE_FILE.COLLOCATION_HINT AS COLLOCATION_HINT,"
":REASON_LOG,"
":RECYCLE_LOG_TIME "
"FROM "
"DUAL,"
"ARCHIVE_FILE "
"WHERE "
"ARCHIVE_FILE.ARCHIVE_FILE_ID = :ARCHIVE_FILE_ID_2";
auto stmt = conn.createStmt(sql);
stmt.bindString(":VID",fileRecycleLog.vid);
stmt.bindUint64(":FSEQ",fileRecycleLog.fseq);
stmt.bindUint64(":BLOCK_ID",fileRecycleLog.blockId);
stmt.bindUint64(":LOGICAL_SIZE_IN_BYTES",fileRecycleLog.logicalSizeInBytes);
stmt.bindUint8(":COPY_NB",fileRecycleLog.copyNb);
stmt.bindUint64(":TAPE_FILE_CREATION_TIME",fileRecycleLog.tapeFileCreationTime);
stmt.bindUint64(":ARCHIVE_FILE_ID",fileRecycleLog.archiveFileId);
stmt.bindString(":REASON_LOG",fileRecycleLog.reasonLog);
stmt.bindUint64(":RECYCLE_LOG_TIME",fileRecycleLog.recycleLogTime);
stmt.bindUint64(":ARCHIVE_FILE_ID_2",fileRecycleLog.archiveFileId);
stmt.executeNonQuery();
insertFileInFileRecycleLog(conn,fileRecycleLog);
}
}
return fileRecycleLogsToInsert;
......@@ -1123,14 +1060,14 @@ void OracleCatalogue::DO_NOT_USE_deleteArchiveFile_DO_NOT_USE(const std::string
//------------------------------------------------------------------------------
// copyArchiveFileToRecycleBinAndDelete
//------------------------------------------------------------------------------
void OracleCatalogue::copyArchiveFileToRecycleBinAndDelete(rdbms::Conn & conn, const common::dataStructures::DeleteArchiveRequest &request, log::LogContext & lc){
void OracleCatalogue::copyArchiveFileToFileRecyleLogAndDelete(rdbms::Conn & conn, const common::dataStructures::DeleteArchiveRequest &request, log::LogContext & lc){
try {
utils::Timer t;
log::TimingList tl;
//We currently do an INSERT INTO, update and two DELETE FROM
//in a single transaction
conn.setAutocommitMode(rdbms::AutocommitMode::AUTOCOMMIT_OFF);
copyArchiveFileToRecycleBin(conn,request);
copyArchiveFileToFileRecycleLog(conn,request);
tl.insertAndReset("insertToRecycleBinTime",t);
setTapeDirty(conn,request.archiveFileID);
tl.insertAndReset("setTapeDirtyTime",t);
......
catalogue/OracleCatalogue.hpp
View file @ cae7b773
......@@ -237,13 +237,13 @@ private:
std::list<cta::catalogue::InsertFileRecycleLog> insertOldCopiesOfFilesIfAnyOnFileRecycleLog(rdbms::Conn & conn);
/**
* Copy the archiveFile and the associated tape files from the ARCHIVE_FILE and TAPE_FILE tables to the recycle-bin tables
* Copy the archiveFile and the associated tape files from the ARCHIVE_FILE and TAPE_FILE tables to the FILE_RECYCLE_LOG table
* and deletes the ARCHIVE_FILE and TAPE_FILE entries.
* @param conn the database connection
* @param request the request that contains the necessary informations to identify the archiveFile to copy to the recycle-bin
* @param request the request that contains the necessary informations to identify the archiveFile to copy to the FILE_RECYCLE_LOG table
* @param lc the log context
*/
void copyArchiveFileToRecycleBinAndDelete(rdbms::Conn & conn, const common::dataStructures::DeleteArchiveRequest &request, log::LogContext & lc) override;
void copyArchiveFileToFileRecyleLogAndDelete(rdbms::Conn & conn, const common::dataStructures::DeleteArchiveRequest &request, log::LogContext & lc) override;
/**
* Delete the TapeFiles and the ArchiveFile from the recycle-bin in one transaction
......
catalogue/PostgresCatalogue.cpp
View file @ cae7b773
......@@ -533,7 +533,7 @@ void PostgresCatalogue::filesWrittenToTape(const std::set<TapeItemWrittenPointer
"DELETE FROM TAPE_FILE WHERE TAPE_FILE.VID = :VID AND TAPE_FILE.FSEQ = :FSEQ";
auto deleteTapeFileStmt = conn.createStmt(deleteTapeFileSql);
deleteTapeFileStmt.bindString(":VID",recycledFile.vid);
deleteTapeFileStmt.bindUint64(":FSEQ",recycledFile.fseq);
deleteTapeFileStmt.bindUint64(":FSEQ",recycledFile.fSeq);
deleteTapeFileStmt.executeNonQuery();
}
......@@ -697,7 +697,7 @@ std::list<cta::catalogue::InsertFileRecycleLog> PostgresCatalogue::insertOldCopi
while(rset.next()){
cta::catalogue::InsertFileRecycleLog fileRecycleLog;
fileRecycleLog.vid = rset.columnString("VID");
fileRecycleLog.fseq = rset.columnUint64("FSEQ");
fileRecycleLog.fSeq = rset.columnUint64("FSEQ");
fileRecycleLog.blockId = rset.columnUint64("BLOCK_ID");
fileRecycleLog.logicalSizeInBytes = rset.columnUint64("LOGICAL_SIZE_IN_BYTES");
fileRecycleLog.copyNb = rset.columnUint8("COPY_NB");
......@@ -710,72 +710,10 @@ std::list<cta::catalogue::InsertFileRecycleLog> PostgresCatalogue::insertOldCopi
}
{
for(auto & fileRecycleLog: fileRecycleLogsToInsert){
const char *const sql =
"INSERT INTO FILE_RECYCLE_LOG("
"FILE_RECYCLE_LOG_ID,"
"VID,"
"FSEQ,"
"BLOCK_ID,"
"LOGICAL_SIZE_IN_BYTES,"
"COPY_NB,"
"TAPE_FILE_CREATION_TIME,"
"ARCHIVE_FILE_ID,"
"DISK_INSTANCE_NAME,"
"DISK_FILE_ID,"
"DISK_FILE_ID_WHEN_DELETED,"
"DISK_FILE_UID,"
"DISK_FILE_GID,"
"SIZE_IN_BYTES,"
"CHECKSUM_BLOB,"
"CHECKSUM_ADLER32,"
"STORAGE_CLASS_ID,"
"ARCHIVE_FILE_CREATION_TIME,"
"RECONCILIATION_TIME,"
"COLLOCATION_HINT,"
"REASON_LOG,"
"RECYCLE_LOG_TIME"
") SELECT "
"nextval('FILE_RECYCLE_LOG_ID_SEQ'),"
":VID,"
":FSEQ,"
":BLOCK_ID,"
":LOGICAL_SIZE_IN_BYTES,"
":COPY_NB,"
":TAPE_FILE_CREATION_TIME,"
":ARCHIVE_FILE_ID,"
"ARCHIVE_FILE.DISK_INSTANCE_NAME,"
"ARCHIVE_FILE.DISK_FILE_ID,"
"ARCHIVE_FILE.DISK_FILE_ID,"
"ARCHIVE_FILE.DISK_FILE_UID,"
"ARCHIVE_FILE.DISK_FILE_GID,"
"ARCHIVE_FILE.SIZE_IN_BYTES,"
"ARCHIVE_FILE.CHECKSUM_BLOB,"
"ARCHIVE_FILE.CHECKSUM_ADLER32,"
"ARCHIVE_FILE.STORAGE_CLASS_ID,"
"ARCHIVE_FILE.CREATION_TIME,"
"ARCHIVE_FILE.RECONCILIATION_TIME,"
"ARCHIVE_FILE.COLLOCATION_HINT,"
":REASON_LOG,"
":RECYCLE_LOG_TIME "
"FROM "
"ARCHIVE_FILE "
"WHERE "
"ARCHIVE_FILE.ARCHIVE_FILE_ID = :ARCHIVE_FILE_ID_2";
auto stmt = conn.createStmt(sql);
stmt.bindString(":VID",fileRecycleLog.vid);
stmt.bindUint64(":FSEQ",fileRecycleLog.fseq);
stmt.bindUint64(":BLOCK_ID",fileRecycleLog.blockId);
stmt.bindUint64(":LOGICAL_SIZE_IN_BYTES",fileRecycleLog.logicalSizeInBytes);
stmt.bindUint8(":COPY_NB",fileRecycleLog.copyNb);
stmt.bindUint64(":TAPE_FILE_CREATION_TIME",fileRecycleLog.tapeFileCreationTime);
stmt.bindUint64(":ARCHIVE_FILE_ID",fileRecycleLog.archiveFileId);
stmt.bindString(":REASON_LOG",fileRecycleLog.reasonLog);
stmt.bindUint64(":RECYCLE_LOG_TIME",fileRecycleLog.recycleLogTime);
stmt.bindUint64(":ARCHIVE_FILE_ID_2",fileRecycleLog.archiveFileId);
stmt.executeNonQuery();
}
insertFileInFileRecycleLog(conn,fileRecycleLog);
}
return fileRecycleLogsToInsert;
}
} catch(exception::Exception &ex) {
ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str());
throw;
......@@ -1080,14 +1018,14 @@ void PostgresCatalogue::beginCreateTemporarySetDeferred(rdbms::Conn &conn) const
//------------------------------------------------------------------------------
// copyArchiveFileToRecycleBinAndDelete
//------------------------------------------------------------------------------
void PostgresCatalogue::copyArchiveFileToRecycleBinAndDelete(rdbms::Conn & conn, const common::dataStructures::DeleteArchiveRequest &request, log::LogContext & lc) {
void PostgresCatalogue::copyArchiveFileToFileRecyleLogAndDelete(rdbms::Conn & conn, const common::dataStructures::DeleteArchiveRequest &request, log::LogContext & lc) {
try {
utils::Timer t;
log::TimingList tl;
//We currently do an INSERT INTO and a DELETE FROM
//in a single transaction
conn.executeNonQuery("BEGIN");
copyArchiveFileToRecycleBin(conn,request);
copyArchiveFileToFileRecycleLog(conn,request);
tl.insertAndReset("insertToRecycleBinTime",t);
setTapeDirty(conn,request.archiveFileID);
tl.insertAndReset("setTapeDirtyTime",t);
......
catalogue/PostgresCatalogue.hpp
View file @ cae7b773
......@@ -271,13 +271,13 @@ private:
void insertTapeFileBatchIntoTempTable(rdbms::Conn &conn, const std::set<TapeFileWritten> &events) const;
/**
* Copy the archiveFile and the associated tape files from the ARCHIVE_FILE and TAPE_FILE tables to the recycle-bin tables
* Copy the archiveFile and the associated tape files from the ARCHIVE_FILE and TAPE_FILE tables to the FILE_RECYCLE_LOG table
* and deletes the ARCHIVE_FILE and TAPE_FILE entries.
* @param conn the database connection
* @param request the request that contains the necessary informations to identify the archiveFile to copy to the recycle-bin
* @param request the request that contains the necessary informations to identify the archiveFile to copy to the FILE_RECYCLE_LOG table
* @param lc the log context
*/
void copyArchiveFileToRecycleBinAndDelete(rdbms::Conn & conn, const common::dataStructures::DeleteArchiveRequest &request, log::LogContext & lc) override;
void copyArchiveFileToFileRecyleLogAndDelete(rdbms::Conn & conn, const common::dataStructures::DeleteArchiveRequest &request, log::LogContext & lc) override;
/**
* Delete the TapeFiles and the ArchiveFile from the recycle-bin in one transaction
......
catalogue/RdbmsCatalogue.cpp
View file @ cae7b773
......@@ -3910,28 +3910,6 @@ std::map<std::string, std::string> &vidToLogicalLibrary) const {
}
}
//------------------------------------------------------------------------------
//getNbNonSupersededFilesOnTape
//------------------------------------------------------------------------------
uint64_t RdbmsCatalogue::getNbNonSupersededFilesOnTape(rdbms::Conn& conn, const std::string& vid) const {
try {
const char *const sql =
"SELECT COUNT(*) AS NB_NON_SUPERSEDED_FILES FROM TAPE_FILE "
"WHERE VID = :VID "
"AND SUPERSEDED_BY_VID IS NULL "
"AND SUPERSEDED_BY_FSEQ IS NULL";
auto stmt = conn.createStmt(sql);
stmt.bindString(":VID", vid);
auto rset = stmt.executeQuery();
rset.next();
return rset.columnUint64("NB_NON_SUPERSEDED_FILES");
} catch(exception::Exception &ex) {
ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str());
throw;
}
}
//------------------------------------------------------------------------------
// getNbFilesOnTape
//------------------------------------------------------------------------------
......@@ -4078,13 +4056,11 @@ void RdbmsCatalogue::reclaimTape(const common::dataStructures::SecurityIdentity
}
}
//The tape exists and is full, we can try to reclaim it
if(getNbNonSupersededFilesOnTape(conn,vid) == 0){
tl.insertAndReset("getNbNonSupersededFilesOnTapeTime",t);
//There is no non-superseded files on the tape, we can reclaim it : delete the files and reset the counters
deleteTapeFiles(conn,vid);
tl.insertAndReset("deleteTapeFilesTime",t);
deleteFilesFromRecycleBin(conn,vid,lc);
tl.insertAndReset("deleteFileFromRecycleBinTime",t);
if(this->getNbFilesOnTape(conn,vid) == 0){
tl.insertAndReset("getNbFilesOnTape",t);
//There is no files on the tape, we can reclaim it : delete the files and reset the counters
deleteFilesFromRecycleLog(conn,vid,lc);
tl.insertAndReset("deleteFileFromRecycleLogTime",t);
resetTapeCounters(conn,admin,vid);
tl.insertAndReset("resetTapeCountersTime",t);
log::ScopedParamContainer spc(lc);
......@@ -7734,7 +7710,7 @@ void RdbmsCatalogue::insertTapeFile(
"FSEQ=:FSEQ";
auto stmt = conn.createStmt(sql);
stmt.bindString(":VID",fileRecycleLog.vid);
stmt.bindUint64(":FSEQ",fileRecycleLog.fseq);
stmt.bindUint64(":FSEQ",fileRecycleLog.fSeq);
stmt.executeNonQuery();
}
}
......@@ -8476,20 +8452,15 @@ void RdbmsCatalogue::updateDiskFileId(const uint64_t archiveFileId, const std::s
//------------------------------------------------------------------------------
// moveArchiveFileToRecycleBin
//------------------------------------------------------------------------------
void RdbmsCatalogue::moveArchiveFileToRecycleBin(const common::dataStructures::DeleteArchiveRequest &request,
void RdbmsCatalogue::moveArchiveFileToRecycleLog(const common::dataStructures::DeleteArchiveRequest &request,
log::LogContext & lc) {
cta::common::dataStructures::ArchiveFile archiveFile;
utils::Timer t, totalTime;
log::TimingList tl;
try{
archiveFile = getArchiveFileById(request.archiveFileID);
tl.insertAndReset("getArchiveFileByIdTime",t);
} catch (cta::exception::Exception &ex){
log::ScopedParamContainer spc(lc);
spc.add("fileId", request.archiveFileID);
lc.log(log::WARNING, "Ignoring request to delete archive file because it does not exist in the catalogue");
if(!request.archiveFile){
//The archive file does not exist in the catalogue, nothing to do with it
return;
}
cta::common::dataStructures::ArchiveFile archiveFile = request.archiveFile.value();
utils::Timer t, totalTime;
log::TimingList tl;
try {
checkDeleteRequestConsistency(request,archiveFile);
tl.insertAndReset("checkDeleteRequestConsistency",t);
......@@ -8518,10 +8489,10 @@ void RdbmsCatalogue::moveArchiveFileToRecycleBin(const common::dataStructures::D
<< " fileSize: " << it->fileSize;
spc.add("TAPE FILE", tapeCopyLogStream.str());
}
lc.log(log::WARNING, "Failed to move archive file to recycle-bin.");
lc.log(log::WARNING, "Failed to move archive file to the file-recycle-log.");
exception::UserError ue;
ue.getMessage() << "Failed to move archive file with ID " << request.archiveFileID << " to the recycle-bin. errorMessage=" << ex.getMessageValue();
ue.getMessage() << "Failed to move archive file with ID " << request.archiveFileID << " to the file-recycle-log. errorMessage=" << ex.getMessageValue();
throw ue;
}
......@@ -8529,8 +8500,8 @@ void RdbmsCatalogue::moveArchiveFileToRecycleBin(const common::dataStructures::D
//All checks are good, we can move the file to the recycle-bin
auto conn = m_connPool.getConn();
rdbms::AutoRollback autoRollback(conn);
copyArchiveFileToRecycleBinAndDelete(conn,request,lc);
tl.insertAndReset("copyArchiveFileToRecycleBinAndDeleteTime",t);
copyArchiveFileToFileRecyleLogAndDelete(conn,request,lc);
tl.insertAndReset("copyArchiveFileToFileRecyleLogAndDeleteTime",t);
tl.insertAndReset("totalTime",totalTime);
log::ScopedParamContainer spc(lc);
spc.add("fileId", std::to_string(request.archiveFileID))
......@@ -8555,7 +8526,7 @@ void RdbmsCatalogue::moveArchiveFileToRecycleBin(const common::dataStructures::D
spc.add("TAPE FILE", tapeCopyLogStream.str());
}
tl.addToLog(spc);
lc.log(log::INFO, "In RdbmsCatalogue::moveArchiveFileToRecycleBin(): ArchiveFile moved to the recycle-bin.");
lc.log(log::INFO, "In RdbmsCatalogue::moveArchiveFileToRecycleLog(): ArchiveFile moved to the file-recycle-log.");
} catch(exception::UserError &) {
throw;
} catch(exception::Exception &ex) {
......@@ -8565,14 +8536,52 @@ void RdbmsCatalogue::moveArchiveFileToRecycleBin(const common::dataStructures::D
}
//------------------------------------------------------------------------------
// insertArchiveFileToRecycleBin
// copyArchiveFileToFileRecycleLog
//------------------------------------------------------------------------------
void RdbmsCatalogue::copyArchiveFileToRecycleBin(rdbms::Conn & conn, const common::dataStructures::DeleteArchiveRequest & request) {
void RdbmsCatalogue::copyArchiveFileToFileRecycleLog(rdbms::Conn & conn, const common::dataStructures::DeleteArchiveRequest & request) {
try{
if(!request.archiveFile){
throw cta::exception::Exception("No archiveFile object has been set in the DeleteArchiveRequest object.");
}
const common::dataStructures::ArchiveFile & archiveFile = request.archiveFile.value();
for(auto &tapeFile: archiveFile.tapeFiles){
//Create one file recycle log entry per tape file
InsertFileRecycleLog fileRecycleLog;
fileRecycleLog.vid = tapeFile.vid;
fileRecycleLog.fSeq = tapeFile.fSeq;
fileRecycleLog.blockId = tapeFile.blockId;
fileRecycleLog.logicalSizeInBytes = tapeFile.fileSize;
fileRecycleLog.copyNb = tapeFile.copyNb;
fileRecycleLog.tapeFileCreationTime = tapeFile.creationTime;
fileRecycleLog.archiveFileId = archiveFile.archiveFileID;
fileRecycleLog.diskFilePath = request.diskFilePath;
fileRecycleLog.reasonLog = InsertFileRecycleLog::getDeletionReasonLog(request.requester.name,request.diskInstance);
fileRecycleLog.recycleLogTime = time(nullptr);
insertFileInFileRecycleLog(conn,fileRecycleLog);