Commit 5df50912 authored by Cedric Caffy's avatar Cedric Caffy
Browse files

[catalogue] Implemented RdbmsCatalogueGetFileRecycleLogItor + unit tests

parent 217e2a28
......@@ -53,6 +53,7 @@ set (CATALOGUE_LIB_SRC_FILES
RdbmsCatalogueGetArchiveFilesItor.cpp
RdbmsCatalogueGetDeletedArchiveFilesItor.cpp
RdbmsCatalogueTapeContentsItor.cpp
RdbmsCatalogueGetFileRecycleLogItor.cpp
SchemaCreatingSqliteCatalogue.cpp
SqliteCatalogue.cpp
SchemaVersion.cpp
......
......@@ -41,6 +41,7 @@
#include "common/dataStructures/DeletedArchiveFile.hpp"
#include "common/dataStructures/DiskFileInfo.hpp"
#include "common/dataStructures/DriveState.hpp"
#include "common/dataStructures/FileRecycleLog.hpp"
#include "common/dataStructures/EntryLog.hpp"
#include "common/dataStructures/ListStorageClassRequest.hpp"
#include "common/dataStructures/LogicalLibrary.hpp"
......@@ -869,6 +870,15 @@ public:
* @return The deleted archive files ordered by archive file ID.
*/
virtual DeletedArchiveFileItor getDeletedArchiveFilesItor() const = 0;
typedef CatalogueItor<common::dataStructures::FileRecycleLog> FileRecycleLogItor;
/**
* Returns all the currently deleted files by looking at the FILE_RECYCLE_LOG table
*
* @return The deleted archive files ordered by archive file ID.
*/
virtual FileRecycleLogItor getFileRecycleLogItor() const = 0;
/**
* Returns the specified files in tape file sequence order.
......@@ -1034,6 +1044,18 @@ public:
* @param lc The log context.
*/
virtual void deleteFileFromRecycleBin(const uint64_t archiveFileId, log::LogContext &lc) = 0;
/**
* Deletes all the log entries corresponding to the vid passed in parameter.
*
* Please note that this method is idempotent. If there are no recycle log
* entries associated to the vid passed in parameter, the method will return
* without any error.
*
* @param vid, the vid of the files to be deleted
* @param lc, the logContext
*/
virtual void deleteFilesFromRecycleLog(const std::string & vid, log::LogContext & lc) = 0;
}; // class Catalogue
......
......@@ -524,9 +524,27 @@ public:
return retryOnLostConnection(m_log, [&]{return m_catalogue->getDeletedArchiveFilesItor();}, m_maxTriesToConnect);
}
FileRecycleLogItor getFileRecycleLogItor() const override {
return retryOnLostConnection(m_log, [&]{return m_catalogue->getFileRecycleLogItor();}, m_maxTriesToConnect);
}
void deleteFileFromRecycleBin(const uint64_t archiveFileId, log::LogContext &lc){
return retryOnLostConnection(m_log,[&]{return m_catalogue->deleteFileFromRecycleBin(archiveFileId,lc);},m_maxTriesToConnect);
}
/**
* Deletes all the log entries corresponding to the vid passed in parameter.
*
* Please note that this method is idempotent. If there are no recycle log
* entries associated to the vid passed in parameter, the method will return
* without any error.
*
* @param vid, the vid of the files to be deleted
* @param lc, the logContext
*/
void deleteFilesFromRecycleLog(const std::string & vid, log::LogContext & lc){
return retryOnLostConnection(m_log,[&]{return m_catalogue->deleteFilesFromRecycleLog(vid,lc);},m_maxTriesToConnect);
}
std::list<common::dataStructures::ArchiveFile> getFilesForRepack(const std::string &vid, const uint64_t startFSeq, const uint64_t maxNbFiles) const override {
return retryOnLostConnection(m_log, [&]{return m_catalogue->getFilesForRepack(vid, startFSeq, maxNbFiles);}, m_maxTriesToConnect);
......
......@@ -28,6 +28,7 @@
#include "common/threading/MutexLocker.hpp"
#include "rdbms/wrapper/ConnFactoryFactory.hpp"
#include "rdbms/Conn.hpp"
#include "InsertFileRecycleLog.hpp"
 
#include <algorithm>
#include <gtest/gtest.h>
......@@ -162,7 +163,7 @@ cta_catalogue_CatalogueTest::cta_catalogue_CatalogueTest():
void cta_catalogue_CatalogueTest::SetUp() {
using namespace cta;
using namespace cta::catalogue;
log::LogContext dummyLc(m_dummyLog);
try {
CatalogueFactory *const *const catalogueFactoryPtrPtr = GetParam();
 
......@@ -213,7 +214,6 @@ void cta_catalogue_CatalogueTest::SetUp() {
}
 
for(const auto &archiveFile: archiveFiles) {
log::LogContext dummyLc(m_dummyLog);
m_catalogue->DO_NOT_USE_deleteArchiveFile_DO_NOT_USE(archiveFile.diskInstance, archiveFile.archiveFileID, dummyLc);
}
}
......@@ -226,10 +226,17 @@ void cta_catalogue_CatalogueTest::SetUp() {
}
for(const auto &deletedArchiveFile: deletedArchiveFiles){
log::LogContext dummyLc(m_dummyLog);
m_catalogue->deleteFileFromRecycleBin(deletedArchiveFile.archiveFileID,dummyLc);
}
}
{
//Delete all the entries from the recycle log table
auto itor = m_catalogue->getFileRecycleLogItor();
std::list<common::dataStructures::FileRecycleLog> filesRecycleLog;
while(itor.hasMore()){
m_catalogue->deleteFilesFromRecycleLog(itor.next().vid,dummyLc);
}
}
{
const std::list<common::dataStructures::Tape> tapes = m_catalogue->getTapes();
for(auto &tape: tapes) {
......@@ -15869,4 +15876,211 @@ TEST_P(cta_catalogue_CatalogueTest, reclaimTapeRemovesFilesFromRecycleBin) {
}
}
 
TEST_P(cta_catalogue_CatalogueTest, emptyFileRecycleLogItorTest) {
using namespace cta;
auto itor = m_catalogue->getFileRecycleLogItor();
ASSERT_FALSE(itor.hasMore());
ASSERT_THROW(itor.next(),cta::exception::Exception);
}
TEST_P(cta_catalogue_CatalogueTest, filesArePutInTheFileRecycleLogInsteadOfBeingSuperseded) {
using namespace cta;
const bool logicalLibraryIsDisabled= false;
const std::string tapePoolName1 = "tape_pool_name_1";
const std::string tapePoolName2 = "tape_pool_name_2";
const uint64_t nbPartialTapes = 1;
const bool isEncrypted = true;
const cta::optional<std::string> supply("value for the supply pool mechanism");
const std::string diskInstance = "disk_instance";
const std::string tapeDrive = "tape_drive";
m_catalogue->createMediaType(m_admin, m_mediaType);
m_catalogue->createLogicalLibrary(m_admin, m_tape1.logicalLibraryName, logicalLibraryIsDisabled, "Create logical library");
m_catalogue->createVirtualOrganization(m_admin, m_vo);
m_catalogue->createTapePool(m_admin, tapePoolName1, m_vo.name, nbPartialTapes, isEncrypted, supply, "Create tape pool");
m_catalogue->createTapePool(m_admin, tapePoolName2, m_vo.name, nbPartialTapes, isEncrypted, supply, "Create tape pool");
m_catalogue->createStorageClass(m_admin, m_storageClassSingleCopy);
auto tape1 = m_tape1;
tape1.tapePoolName = tapePoolName1;
auto tape2 = m_tape2;
tape2.tapePoolName = tapePoolName2;
m_catalogue->createTape(m_admin, tape1);
m_catalogue->createTape(m_admin, tape2);
ASSERT_FALSE(m_catalogue->getArchiveFilesItor().hasMore());
const uint64_t nbArchiveFiles = 10; // Must be a multiple of 2 for this test
const uint64_t archiveFileSize = 2 * 1000 * 1000 * 1000;
std::set<catalogue::TapeItemWrittenPointer> tapeFilesWrittenCopy1;
for(uint64_t i = 1; i <= nbArchiveFiles; i++) {
std::ostringstream diskFileId;
diskFileId << (12345677 + i);
std::ostringstream diskFilePath;
diskFilePath << "/test/file"<<i;
// Tape copy 1 written to tape
auto fileWrittenUP=cta::make_unique<cta::catalogue::TapeFileWritten>();
auto & fileWritten = *fileWrittenUP;
fileWritten.archiveFileId = i;
fileWritten.diskInstance = diskInstance;
fileWritten.diskFileId = diskFileId.str();
fileWritten.diskFilePath = diskFilePath.str();
fileWritten.diskFileOwnerUid = PUBLIC_DISK_USER;
fileWritten.diskFileGid = PUBLIC_DISK_GROUP;
fileWritten.size = archiveFileSize;
fileWritten.checksumBlob.insert(checksum::ADLER32, "1357");
fileWritten.storageClassName = m_storageClassSingleCopy.name;
fileWritten.vid = tape1.vid;
fileWritten.fSeq = i;
fileWritten.blockId = i * 100;
fileWritten.copyNb = 1;
fileWritten.tapeDrive = tapeDrive;
tapeFilesWrittenCopy1.emplace(fileWrittenUP.release());
}
m_catalogue->filesWrittenToTape(tapeFilesWrittenCopy1);
{
ASSERT_TRUE(m_catalogue->getArchiveFilesItor().hasMore());
ASSERT_FALSE(m_catalogue->getFileRecycleLogItor().hasMore());
}
log::LogContext dummyLc(m_dummyLog);
//Archive the same files but in a new tape
for(auto & tapeItemWritten: tapeFilesWrittenCopy1){
cta::catalogue::TapeFileWritten * tapeItem = static_cast<cta::catalogue::TapeFileWritten *>(tapeItemWritten.get());
tapeItem->vid = tape2.vid;
}
m_catalogue->filesWrittenToTape(tapeFilesWrittenCopy1);
//Change the vid back to the first one to test the content of the file recycle log afterwards
for(auto & tapeItemWritten: tapeFilesWrittenCopy1){
cta::catalogue::TapeFileWritten * tapeItem = static_cast<cta::catalogue::TapeFileWritten *>(tapeItemWritten.get());
tapeItem->vid = tape1.vid;
}
//Check that the new files written exist on the catalogue
{
auto archiveFilesItor = m_catalogue->getArchiveFilesItor();
bool hasArchiveFilesItor = false;
while(archiveFilesItor.hasMore()){
hasArchiveFilesItor = true;
//The vid is the destination one
ASSERT_EQ(tape2.vid,archiveFilesItor.next().tapeFiles.at(1).vid);
}
ASSERT_TRUE(hasArchiveFilesItor);
}
//Check that the old files are in the file recycle logs
std::list<common::dataStructures::FileRecycleLog> fileRecycleLogs;
{
auto fileRecycleLogItor = m_catalogue->getFileRecycleLogItor();
while(fileRecycleLogItor.hasMore()){
fileRecycleLogs.push_back(fileRecycleLogItor.next());
}
ASSERT_FALSE(fileRecycleLogs.empty());
//Now we check the consistency of what is returned by the file recycle log
for(auto fileRecycleLog: fileRecycleLogs){
//First, get the correct file written to tape associated to the current fileRecycleLog
auto tapeFilesWrittenCopy1Itor = std::find_if(tapeFilesWrittenCopy1.begin(), tapeFilesWrittenCopy1.end(),[fileRecycleLog](const cta::catalogue::TapeItemWrittenPointer & item){
cta::catalogue::TapeFileWritten * fileWrittenPtr = static_cast<cta::catalogue::TapeFileWritten *>(item.get());
return fileWrittenPtr->archiveFileId == fileRecycleLog.archiveFileId;
});
ASSERT_NE(tapeFilesWrittenCopy1.end(), tapeFilesWrittenCopy1Itor);
cta::catalogue::TapeFileWritten * fileWrittenPtr = static_cast<cta::catalogue::TapeFileWritten *>(tapeFilesWrittenCopy1Itor->get());
ASSERT_EQ(fileRecycleLog.vid,tape1.vid);
ASSERT_EQ(fileRecycleLog.fSeq,fileWrittenPtr->fSeq);
ASSERT_EQ(fileRecycleLog.blockId,fileWrittenPtr->blockId);
ASSERT_EQ(fileRecycleLog.logicalSizeInBytes,fileWrittenPtr->size);
ASSERT_EQ(fileRecycleLog.copyNb,fileWrittenPtr->copyNb);
ASSERT_EQ(fileRecycleLog.archiveFileId,fileWrittenPtr->archiveFileId);
ASSERT_EQ(fileRecycleLog.diskInstanceName,fileWrittenPtr->diskInstance);
ASSERT_EQ(fileRecycleLog.diskFileId,fileWrittenPtr->diskFileId);
ASSERT_EQ(fileRecycleLog.diskFileIdWhenDeleted,fileWrittenPtr->diskFileId);
ASSERT_EQ(fileRecycleLog.diskFileUid,fileWrittenPtr->diskFileOwnerUid);
ASSERT_EQ(fileRecycleLog.diskFileGid,fileWrittenPtr->diskFileGid);
ASSERT_EQ(fileRecycleLog.sizeInBytes,fileWrittenPtr->size);
ASSERT_EQ(fileRecycleLog.checksumBlob,fileWrittenPtr->checksumBlob);
ASSERT_EQ(fileRecycleLog.storageClassName,fileWrittenPtr->storageClassName);
ASSERT_EQ(fileRecycleLog.reconciliationTime,fileRecycleLog.archiveFileCreationTime);
ASSERT_EQ(cta::nullopt, fileRecycleLog.collocationHint);
ASSERT_EQ(cta::nullopt, fileRecycleLog.diskFilePath);\
ASSERT_EQ(cta::catalogue::InsertFileRecycleLog::getRepackReasonLog(),fileRecycleLog.reasonLog);
}
}
}
TEST_P(cta_catalogue_CatalogueTest, sameFileWrittenToSameTapePutThePreviousCopyOnTheFileRecycleLog) {
using namespace cta;
const bool logicalLibraryIsDisabled= false;
const std::string tapePoolName1 = "tape_pool_name_1";
const std::string tapePoolName2 = "tape_pool_name_2";
const uint64_t nbPartialTapes = 1;
const bool isEncrypted = true;
const cta::optional<std::string> supply("value for the supply pool mechanism");
const std::string diskInstance = "disk_instance";
const std::string tapeDrive = "tape_drive";
m_catalogue->createMediaType(m_admin, m_mediaType);
m_catalogue->createLogicalLibrary(m_admin, m_tape1.logicalLibraryName, logicalLibraryIsDisabled, "Create logical library");
m_catalogue->createVirtualOrganization(m_admin, m_vo);
m_catalogue->createTapePool(m_admin, tapePoolName1, m_vo.name, nbPartialTapes, isEncrypted, supply, "Create tape pool");
m_catalogue->createTapePool(m_admin, tapePoolName2, m_vo.name, nbPartialTapes, isEncrypted, supply, "Create tape pool");
m_catalogue->createStorageClass(m_admin, m_storageClassSingleCopy);
auto tape1 = m_tape1;
tape1.tapePoolName = tapePoolName1;
m_catalogue->createTape(m_admin, tape1);
ASSERT_FALSE(m_catalogue->getArchiveFilesItor().hasMore());
const uint64_t archiveFileSize = 2 * 1000 * 1000 * 1000;
std::set<catalogue::TapeItemWrittenPointer> tapeFilesWrittenCopy1;
std::ostringstream diskFileId;
diskFileId << 12345677;
std::ostringstream diskFilePath;
diskFilePath << "/test/file1";
// Two files same archiveFileId and CopyNb on the same tape
auto fileWrittenUP=cta::make_unique<cta::catalogue::TapeFileWritten>();
auto & fileWritten = *fileWrittenUP;
fileWritten.archiveFileId = 1;
fileWritten.diskInstance = diskInstance;
fileWritten.diskFileId = diskFileId.str();
fileWritten.diskFilePath = diskFilePath.str();
fileWritten.diskFileOwnerUid = PUBLIC_DISK_USER;
fileWritten.diskFileGid = PUBLIC_DISK_GROUP;
fileWritten.size = archiveFileSize;
fileWritten.checksumBlob.insert(checksum::ADLER32, "1357");
fileWritten.storageClassName = m_storageClassSingleCopy.name;
fileWritten.vid = tape1.vid;
fileWritten.fSeq = 1;
fileWritten.blockId = 1 * 100;
fileWritten.copyNb = 1;
fileWritten.tapeDrive = tapeDrive;
tapeFilesWrittenCopy1.emplace(fileWrittenUP.release());
m_catalogue->filesWrittenToTape(tapeFilesWrittenCopy1);
auto & tapeItemWritten = *(tapeFilesWrittenCopy1.begin());
cta::catalogue::TapeFileWritten * tapeItem = static_cast<cta::catalogue::TapeFileWritten *>(tapeItemWritten.get());
tapeItem->fSeq = 2;
tapeItem->blockId = 2 *100 + 1;
m_catalogue->filesWrittenToTape(tapeFilesWrittenCopy1);
{
auto archiveFilesItor = m_catalogue->getArchiveFilesItor();
ASSERT_TRUE(archiveFilesItor.hasMore());
//The file with fseq 2 is on the active archive files of CTA
ASSERT_EQ(2,archiveFilesItor.next().tapeFiles.at(1).fSeq);
auto fileRecycleLogItor = m_catalogue->getFileRecycleLogItor();
ASSERT_TRUE(fileRecycleLogItor.hasMore());
//The previous file (fSeq = 1) is on the recycle log
ASSERT_EQ(1,fileRecycleLogItor.next().fSeq);
}
}
} // namespace unitTests
......@@ -83,7 +83,9 @@ public:
common::dataStructures::ArchiveFile getArchiveFileById(const uint64_t id) const override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
ArchiveFileItor getArchiveFilesItor(const TapeFileSearchCriteria& searchCriteria) const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
DeletedArchiveFileItor getDeletedArchiveFilesItor() const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
FileRecycleLogItor getFileRecycleLogItor() const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
void deleteFileFromRecycleBin(const uint64_t archiveFileId, log::LogContext &lc) {throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented");}
void deleteFilesFromRecycleLog(const std::string & vid, log::LogContext & lc) {throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented");}
std::list<common::dataStructures::ArchiveFile> getFilesForRepack(const std::string &vid, const uint64_t startFSeq, const uint64_t maxNbFiles) const override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
ArchiveFileItor getArchiveFilesForRepackItor(const std::string &vid, const uint64_t startFSeq) const override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
std::list<common::dataStructures::ArchiveRoute> getArchiveRoutes() const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
......
......@@ -25,29 +25,22 @@
namespace cta { namespace catalogue {
struct InsertFileRecycleLog {
class InsertFileRecycleLog {
public:
std::string vid;
uint64_t fseq;
uint64_t blockId;
uint64_t logicalSizeInBytes;
uint64_t copyNb;
uint8_t copyNb;
time_t tapeFileCreationTime;
uint64_t archiveFileId;
std::string diskInstanceName;
uint64_t diskFileId;
uint64_t diskFileIdWhenDeleted;
uint64_t diskFileUid;
uint64_t diskFileGid;
uint64_t sizeInBytes;
checksum::ChecksumBlob checksumBlob;
uint32_t checksumAdler32;
uint64_t storageClassId;
time_t archiveFileCreationTime;
time_t reconciliationTime;
std::string collocationHint;
std::string diskFilePath;
std::string reasonLog;
time_t recycleLogTime;
static std::string getRepackReasonLog(){
return "REPACK";
}
};
}}
......
......@@ -572,6 +572,8 @@ void OracleCatalogue::filesWrittenToTape(const std::set<TapeItemWrittenPointer>
fileSizeAndChecksum.checksumBlob.validate(event.checksumBlob);
}
std::list<InsertFileRecycleLog> recycledFiles = insertOldCopiesOfFilesIfAnyOnFileRecycleLog(conn);
{
const char *const sql =
"INSERT INTO TAPE_FILE (VID, FSEQ, BLOCK_ID, LOGICAL_SIZE_IN_BYTES," "\n"
......@@ -582,8 +584,6 @@ void OracleCatalogue::filesWrittenToTape(const std::set<TapeItemWrittenPointer>
stmt.executeNonQuery();
}
std::list<InsertFileRecycleLog> recycledFiles = insertOldCopiesOfFilesIfAnyOnFileRecycleLog(conn);
for(auto & recycledFile: recycledFiles){
const char *const sql =
"DELETE FROM "
......@@ -782,7 +782,7 @@ std::list<cta::catalogue::InsertFileRecycleLog> OracleCatalogue::insertOldCopies
"ON "
"TEMP_TAPE_FILE_INSERTION_BATCH.ARCHIVE_FILE_ID = TAPE_FILE.ARCHIVE_FILE_ID AND TEMP_TAPE_FILE_INSERTION_BATCH.COPY_NB = TAPE_FILE.COPY_NB "
"WHERE "
"TAPE_FILE.VID != TEMP_TAPE_FILE_INSERTION_BATCH.VID";
"TAPE_FILE.VID != TEMP_TAPE_FILE_INSERTION_BATCH.VID OR TAPE_FILE.FSEQ != TEMP_TAPE_FILE_INSERTION_BATCH.FSEQ";
auto stmt = conn.createStmt(sql);
auto rset = stmt.executeQuery();
while(rset.next()){
......@@ -794,7 +794,7 @@ std::list<cta::catalogue::InsertFileRecycleLog> OracleCatalogue::insertOldCopies
fileRecycleLog.copyNb = rset.columnUint8("COPY_NB");
fileRecycleLog.tapeFileCreationTime = rset.columnUint64("TAPE_FILE_CREATION_TIME");
fileRecycleLog.archiveFileId = rset.columnUint64("ARCHIVE_FILE_ID");
fileRecycleLog.reasonLog = "Repacked from VID " + fileRecycleLog.vid;
fileRecycleLog.reasonLog = InsertFileRecycleLog::getRepackReasonLog();
fileRecycleLog.recycleLogTime = time(nullptr);
fileRecycleLogsToInsert.push_back(fileRecycleLog);
}
......
......@@ -502,11 +502,7 @@ void PostgresCatalogue::filesWrittenToTape(const std::set<TapeItemWrittenPointer
"-- :LOGICAL_SIZE_IN_BYTES," "\n"
"-- :COPY_NB," "\n"
"-- :CREATION_TIME," "\n"
"-- :ARCHIVE_FILE_ID" "\n"
"INSERT INTO TAPE_FILE (VID, FSEQ, BLOCK_ID, LOGICAL_SIZE_IN_BYTES," "\n"
"COPY_NB, CREATION_TIME, ARCHIVE_FILE_ID) " "\n"
"SELECT VID, FSEQ, BLOCK_ID, LOGICAL_SIZE_IN_BYTES," "\n"
"COPY_NB, CREATION_TIME, ARCHIVE_FILE_ID FROM TEMP_TAPE_FILE_INSERTION_BATCH;" "\n";
"-- :ARCHIVE_FILE_ID;" "\n";
auto stmt = conn.createStmt(sql);
rdbms::wrapper::PostgresStmt &postgresStmt = dynamic_cast<rdbms::wrapper::PostgresStmt &>(stmt.getStmt());
......@@ -522,6 +518,16 @@ void PostgresCatalogue::filesWrittenToTape(const std::set<TapeItemWrittenPointer
auto recycledFiles = insertOldCopiesOfFilesIfAnyOnFileRecycleLog(conn);
{
//Insert the tapefiles from the TEMP_TAPE_FILE_INSERTION_BATCH
const char * const insertTapeFileSql =
"INSERT INTO TAPE_FILE (VID, FSEQ, BLOCK_ID, LOGICAL_SIZE_IN_BYTES," "\n"
"COPY_NB, CREATION_TIME, ARCHIVE_FILE_ID) " "\n"
"SELECT VID, FSEQ, BLOCK_ID, LOGICAL_SIZE_IN_BYTES," "\n"
"COPY_NB, CREATION_TIME, ARCHIVE_FILE_ID FROM TEMP_TAPE_FILE_INSERTION_BATCH;" "\n";
conn.executeNonQuery(insertTapeFileSql);
}
for(auto & recycledFile: recycledFiles){
const char * const deleteTapeFileSql =
"DELETE FROM TAPE_FILE WHERE TAPE_FILE.VID = :VID AND TAPE_FILE.FSEQ = :FSEQ";
......@@ -685,7 +691,7 @@ std::list<cta::catalogue::InsertFileRecycleLog> PostgresCatalogue::insertOldCopi
"ON "
"TEMP_TAPE_FILE_INSERTION_BATCH.ARCHIVE_FILE_ID = TAPE_FILE.ARCHIVE_FILE_ID AND TEMP_TAPE_FILE_INSERTION_BATCH.COPY_NB = TAPE_FILE.COPY_NB "
"WHERE "
"TAPE_FILE.VID != TEMP_TAPE_FILE_INSERTION_BATCH.VID";
"TAPE_FILE.VID != TEMP_TAPE_FILE_INSERTION_BATCH.VID OR TAPE_FILE.FSEQ != TEMP_TAPE_FILE_INSERTION_BATCH.FSEQ";
auto stmt = conn.createStmt(sql);
auto rset = stmt.executeQuery();
while(rset.next()){
......@@ -697,7 +703,7 @@ std::list<cta::catalogue::InsertFileRecycleLog> PostgresCatalogue::insertOldCopi
fileRecycleLog.copyNb = rset.columnUint8("COPY_NB");
fileRecycleLog.tapeFileCreationTime = rset.columnUint64("TAPE_FILE_CREATION_TIME");
fileRecycleLog.archiveFileId = rset.columnUint64("ARCHIVE_FILE_ID");
fileRecycleLog.reasonLog = "Repacked from VID " + fileRecycleLog.vid;
fileRecycleLog.reasonLog = InsertFileRecycleLog::getRepackReasonLog();
fileRecycleLog.recycleLogTime = time(nullptr);
fileRecycleLogsToInsert.push_back(fileRecycleLog);
}
......
......@@ -37,6 +37,7 @@
#include "common/utils/utils.hpp"
#include "rdbms/AutoRollback.hpp"
#include "version.h"
#include "RdbmsCatalogueGetFileRecycleLogItor.hpp"
#include <ctype.h>
#include <memory>
......@@ -476,7 +477,7 @@ void RdbmsCatalogue::modifyVirtualOrganizationName(const common::dataStructures:
"LAST_UPDATE_HOST_NAME = :LAST_UPDATE_HOST_NAME,"
"LAST_UPDATE_TIME = :LAST_UPDATE_TIME "
"WHERE "
"VIRTUAL_ORGANIZATION_NAME = :CURRENT_VIRTUAL_ORGANIZATION_NAME";
"VIRTUAL_ORGANIZATION_NAME = :CUR_VIRTUAL_ORGANIZATION_NAME";
auto conn = m_connPool.getConn();
if(newVoName != currentVoName){
if(virtualOrganizationExists(conn,newVoName)){
......@@ -488,7 +489,7 @@ void RdbmsCatalogue::modifyVirtualOrganizationName(const common::dataStructures:
stmt.bindString(":LAST_UPDATE_USER_NAME", admin.username);
stmt.bindString(":LAST_UPDATE_HOST_NAME", admin.host);
stmt.bindUint64(":LAST_UPDATE_TIME", now);
stmt.bindString(":CURRENT_VIRTUAL_ORGANIZATION_NAME", currentVoName);
stmt.bindString(":CUR_VIRTUAL_ORGANIZATION_NAME", currentVoName);
stmt.executeNonQuery();
if(0 == stmt.getNbAffectedRows()) {
......@@ -6801,6 +6802,18 @@ Catalogue::DeletedArchiveFileItor RdbmsCatalogue::getDeletedArchiveFilesItor() c
}
}
Catalogue::FileRecycleLogItor RdbmsCatalogue::getFileRecycleLogItor() const {
try {
auto impl = new RdbmsCatalogueGetFileRecycleLogItor(m_log, m_archiveFileListingConnPool);
return FileRecycleLogItor(impl);
} catch(exception::UserError &) {
throw;
} catch(exception::Exception &ex) {
ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str());
throw;
}
}
//------------------------------------------------------------------------------
// getFilesForRepack
//------------------------------------------------------------------------------
......@@ -7679,120 +7692,51 @@ void RdbmsCatalogue::insertTapeFile(
const uint64_t archiveFileId) {
rdbms::AutoRollback autoRollback(conn);
try{
std::list<InsertFileRecycleLog> insertedFilesRecycleLog = insertOldCopiesOfFilesIfAnyOnFileRecycleLog(conn,tapeFile,archiveFileId);
{
const time_t now = time(nullptr);
const char *const sql =
"INSERT INTO TAPE_FILE("
"VID,"
"FSEQ,"
"BLOCK_ID,"
"LOGICAL_SIZE_IN_BYTES,"
"COPY_NB,"
"CREATION_TIME,"
"ARCHIVE_FILE_ID)"
"VALUES("
":VID,"
":FSEQ,"
":BLOCK_ID,"
":LOGICAL_SIZE_IN_BYTES,"
":COPY_NB,"
":CREATION_TIME,"
":ARCHIVE_FILE_ID)";
auto stmt = conn.createStmt(sql);
stmt.bindString(":VID", tapeFile.vid);
stmt.bindUint64(":FSEQ", tapeFile.fSeq);
stmt.bindUint64(":BLOCK_ID", tapeFile.blockId);
stmt.bindUint64(":LOGICAL_SIZE_IN_BYTES", tapeFile.fileSize);
stmt.bindUint64(":COPY_NB", tapeFile.copyNb);
stmt.bindUint64(":CREATION_TIME", now);
stmt.bindUint64(":ARCHIVE_FILE_ID", archiveFileId);
stmt.executeNonQuery();
}
{
const time_t now = time(nullptr);
const char *const sql =
"INSERT INTO TAPE_FILE("
"VID,"
"FSEQ,"
"BLOCK_ID,"
"LOGICAL_SIZE_IN_BYTES,"
"COPY_NB,"
"CREATION_TIME,"
"ARCHIVE_FILE_ID)"
"VALUES("
":VID,"
":FSEQ,"
":BLOCK_ID,"
":LOGICAL_SIZE_IN_BYTES,"
":COPY_NB,"
":CREATION_TIME,"
":ARCHIVE_FILE_ID)";
auto stmt = conn.createStmt(sql);
stmt.bindString(":VID", tapeFile.vid);
stmt.bindUint64(":FSEQ", tapeFile.fSeq);
stmt.bindUint64(":BLOCK_ID", tapeFile.blockId);
stmt.bindUint64(":LOGICAL_SIZE_IN_BYTES", tapeFile.fileSize);
stmt.bindUint64(":COPY_NB", tapeFile.copyNb);
stmt.bindUint64(":CREATION_TIME", now);
stmt.bindUint64(":ARCHIVE_FILE_ID", archiveFileId);
stmt.executeNonQuery();
}
{
uint64_t fileRecycleLogId = getNextFileRecyleLogId(conn);
const char *const sql =
"INSERT INTO FILE_RECYCLE_LOG ("
"FILE_RECYCLE_LOG_ID,"
"VID,"
"FSEQ,"
"BLOCK_ID,"
"LOGICAL_SIZE_IN_BYTES,"
"COPY_NB,"
"TAPE_FILE_CREATION_TIME,"
"ARCHIVE_FILE_ID,"
"DISK_INSTANCE_NAME,"
"DISK_FILE_ID,"
"DISK_FILE_ID_WHEN_DELETED,"
"DISK_FILE_UID,"
"DISK_FILE_GID,"
"SIZE_IN_BYTES,"
"CHECKSUM_BLOB,"
"CHECKSUM_ADLER32,"
"STORAGE_CLASS_ID,"
"ARCHIVE_FILE_CREATION_TIME,"
"RECONCILIATION_TIME,"
"COLLOCATION_HINT,"
"REASON_LOG,"
"RECYCLE_LOG_TIME"
") SELECT "
":FILE_RECYCLE_LOG_ID,"
"TAPE_FILE.VID,"
"TAPE_FILE.FSEQ,"
"TAPE_FILE.BLOCK_ID,"
"TAPE_FILE.LOGICAL_SIZE_IN_BYTES,"
"TAPE_FILE.COPY_NB,"
"TAPE_FILE.CREATION_TIME,"
"TAPE_FILE.ARCHIVE_FILE_ID,"
"ARCHIVE_FILE.DISK_INSTANCE_NAME AS DISK_INSTANCE_NAME,"
"ARCHIVE_FILE.DISK_FILE_ID AS DISK_FILE_ID,"
"ARCHIVE_FILE.DISK_FILE_ID AS DISK_FILE_ID_2,"
"ARCHIVE_FILE.DISK_FILE_UID AS DISK_FILE_UID,"
"ARCHIVE_FILE.DISK_FILE_GID AS DISK_FILE_GID,"
"ARCHIVE_FILE.SIZE_IN_BYTES AS SIZE_IN_BYTES,"
"ARCHIVE_FILE.CHECKSUM_BLOB AS CHECKSUM_BLOB,"
"ARCHIVE_FILE.CHECKSUM_ADLER32 AS CHECKSUM_ADLER32,"
"ARCHIVE_FILE.STORAGE_CLASS_ID AS STORAGE_CLASS_ID,"
"ARCHIVE_FILE.CREATION_TIME AS ARCHIVE_FILE_CREATION_TIME,"
"ARCHIVE_FILE.RECONCILIATION_TIME AS RECONCILIATION_TIME,"
"ARCHIVE_FILE.COLLOCATION_HINT AS COLLOCATION_HINT,"
":REASON_LOG,"
":RECYCLE_LOG_TIME "
"FROM "
"ARCHIVE_FILE "
"JOIN TAPE_FILE ON "
"ARCHIVE_FILE.ARCHIVE_FILE_ID = TAPE_FILE.ARCHIVE_FILE_ID AND TAPE_FILE.COPY_NB=:COPY_NB AND TAPE_FILE.VID <> :NEW_VID AND TAPE_FILE.FSEQ <> :NEW_FSEQ "