Commit 025419a3 authored by Steven Murray's avatar Steven Murray
Browse files

Second version of Catalogue::getFilesForRepack()

parent 62f4c772
......@@ -493,15 +493,15 @@ public:
const TapeFileSearchCriteria &searchCriteria = TapeFileSearchCriteria()) const = 0;
/**
* Returns the specified tape files in file sequence order.
* Returns the specified files in tape file sequence order.
*
* @param vid The volume identifier of the tape.
* @param startFSeq The file sequence number of the first file. Please note
* that there might not be a file with this exact file sequence number.
* @param maxNbFiles The maximum number of files to be returned.
* @return The tape files.
* @return The specified files in tape file sequence order.
*/
virtual std::list<common::dataStructures::TapeFile> getTapeFilesForRepack(
virtual std::list<common::dataStructures::ArchiveFile> getFilesForRepack(
const std::string &vid,
const uint64_t startFSeq,
const uint64_t maxNbFiles) const = 0;
......
......@@ -341,8 +341,8 @@ public:
return retryOnLostConnection(m_log, [&]{return m_catalogue->getArchiveFiles(searchCriteria);}, m_maxTriesToConnect);
}
std::list<common::dataStructures::TapeFile> getTapeFilesForRepack(const std::string &vid, const uint64_t startFSeq, const uint64_t maxNbFiles) const override {
return retryOnLostConnection(m_log, [&]{return m_catalogue->getTapeFilesForRepack(vid, startFSeq, maxNbFiles);}, m_maxTriesToConnect);
std::list<common::dataStructures::ArchiveFile> getFilesForRepack(const std::string &vid, const uint64_t startFSeq, const uint64_t maxNbFiles) const override {
return retryOnLostConnection(m_log, [&]{return m_catalogue->getFilesForRepack(vid, startFSeq, maxNbFiles);}, m_maxTriesToConnect);
}
common::dataStructures::ArchiveFileSummary getTapeFileSummary(const TapeFileSearchCriteria &searchCriteria = TapeFileSearchCriteria()) const override {
......
......@@ -209,6 +209,29 @@ std::map<uint64_t, cta::common::dataStructures::ArchiveFile> cta_catalogue_Catal
}
}
//------------------------------------------------------------------------------
// archiveFileListToMap
//------------------------------------------------------------------------------
std::map<uint64_t, cta::common::dataStructures::ArchiveFile> cta_catalogue_CatalogueTest::archiveFileListToMap(
const std::list<cta::common::dataStructures::ArchiveFile> &listOfArchiveFiles) {
using namespace cta;
try {
std::map<uint64_t, common::dataStructures::ArchiveFile> archiveIdToArchiveFile;
for (auto &archiveFile: listOfArchiveFiles) {
if(archiveIdToArchiveFile.end() != archiveIdToArchiveFile.find(archiveFile.archiveFileID)) {
throw exception::Exception(std::string("Duplicate archive file ID: value=") + std::to_string(archiveFile.archiveFileID));
}
archiveIdToArchiveFile[archiveFile.archiveFileID] = archiveFile;
}
return archiveIdToArchiveFile;
} catch(exception::Exception &ex) {
throw exception::Exception(std::string(__FUNCTION__) + " failed: " + ex.getMessage().str());
}
}
//------------------------------------------------------------------------------
// adminUserListToMap
//------------------------------------------------------------------------------
......@@ -7237,7 +7260,7 @@ TEST_P(cta_catalogue_CatalogueTest, filesWrittenToTape_many_archive_files) {
const std::string tapeDrive = "tape_drive";
ASSERT_FALSE(m_catalogue->getArchiveFiles().hasMore());
const uint64_t nbArchiveFiles = 10;
const uint64_t nbArchiveFiles = 10; // Must be a multiple of 2 fo rthis test
const uint64_t archiveFileSize = 2 * 1000 * 1000 * 1000;
const uint64_t compressedFileSize = archiveFileSize;
......@@ -7479,6 +7502,204 @@ TEST_P(cta_catalogue_CatalogueTest, filesWrittenToTape_many_archive_files) {
}
}
for(uint64_t copyNb = 1; copyNb <= 2; copyNb++) {
const std::string vid = copyNb == 1 ? vid1 : vid2;
const uint64_t startFseq = 1;
const uint64_t maxNbFiles = nbArchiveFiles;
auto archiveFiles = m_catalogue->getFilesForRepack(vid, startFseq, maxNbFiles);
auto m = archiveFileListToMap(archiveFiles);
ASSERT_EQ(nbArchiveFiles, m.size());
for(uint64_t i = 1; i <= nbArchiveFiles; i++) {
std::ostringstream diskFileId;
diskFileId << (12345677 + i);
std::ostringstream diskFilePath;
diskFilePath << "/public_dir/public_file_" << i;
catalogue::TapeFileWritten fileWritten;
fileWritten.archiveFileId = i;
fileWritten.diskInstance = storageClass.diskInstance;
fileWritten.diskFileId = diskFileId.str();
fileWritten.diskFilePath = diskFilePath.str();
fileWritten.diskFileUser = "public_disk_user";
fileWritten.diskFileGroup = "public_disk_group";
fileWritten.diskFileRecoveryBlob = "opaque_disk_file_recovery_contents";
fileWritten.size = archiveFileSize;
fileWritten.checksumType = checksumType;
fileWritten.checksumValue = checksumValue;
fileWritten.storageClassName = storageClass.name;
fileWritten.vid = vid;
fileWritten.fSeq = i;
fileWritten.blockId = i * 100;
fileWritten.compressedSize = compressedFileSize;
fileWritten.copyNb = copyNb;
const auto idAndFile = m.find(i);
ASSERT_FALSE(m.end() == idAndFile);
const common::dataStructures::ArchiveFile archiveFile = idAndFile->second;
ASSERT_EQ(fileWritten.archiveFileId, archiveFile.archiveFileID);
ASSERT_EQ(fileWritten.diskInstance, archiveFile.diskInstance);
ASSERT_EQ(fileWritten.diskFileId, archiveFile.diskFileId);
ASSERT_EQ(fileWritten.diskFilePath, archiveFile.diskFileInfo.path);
ASSERT_EQ(fileWritten.diskFileUser, archiveFile.diskFileInfo.owner);
ASSERT_EQ(fileWritten.diskFileGroup, archiveFile.diskFileInfo.group);
ASSERT_EQ(fileWritten.diskFileRecoveryBlob, archiveFile.diskFileInfo.recoveryBlob);
ASSERT_EQ(fileWritten.size, archiveFile.fileSize);
ASSERT_EQ(fileWritten.checksumType, archiveFile.checksumType);
ASSERT_EQ(fileWritten.checksumValue, archiveFile.checksumValue);
ASSERT_EQ(fileWritten.storageClassName, archiveFile.storageClass);
// There is only one tape copy because repack only want the tape file on a
// single tape
ASSERT_EQ(1, archiveFile.tapeFiles.size());
{
const auto it = archiveFile.tapeFiles.find(copyNb);
ASSERT_NE(archiveFile.tapeFiles.end(), it);
ASSERT_EQ(fileWritten.vid, it->second.vid);
ASSERT_EQ(fileWritten.fSeq, it->second.fSeq);
ASSERT_EQ(fileWritten.blockId, it->second.blockId);
ASSERT_EQ(fileWritten.compressedSize, it->second.compressedSize);
ASSERT_EQ(fileWritten.checksumType, it->second.checksumType);
ASSERT_EQ(fileWritten.checksumValue, it->second.checksumValue);
ASSERT_EQ(fileWritten.copyNb, it->second.copyNb);
ASSERT_EQ(fileWritten.copyNb, it->first);
}
}
}
for(uint64_t copyNb = 1; copyNb <= 2; copyNb++) {
const std::string vid = copyNb == 1 ? vid1 : vid2;
const uint64_t startFseq = 1;
const uint64_t maxNbFiles = nbArchiveFiles / 2;
auto archiveFiles = m_catalogue->getFilesForRepack(vid, startFseq, maxNbFiles);
auto m = archiveFileListToMap(archiveFiles);
ASSERT_EQ(nbArchiveFiles, m.size());
for(uint64_t i = 1; i <= nbArchiveFiles / 2; i++) {
std::ostringstream diskFileId;
diskFileId << (12345677 + i);
std::ostringstream diskFilePath;
diskFilePath << "/public_dir/public_file_" << i;
catalogue::TapeFileWritten fileWritten;
fileWritten.archiveFileId = i;
fileWritten.diskInstance = storageClass.diskInstance;
fileWritten.diskFileId = diskFileId.str();
fileWritten.diskFilePath = diskFilePath.str();
fileWritten.diskFileUser = "public_disk_user";
fileWritten.diskFileGroup = "public_disk_group";
fileWritten.diskFileRecoveryBlob = "opaque_disk_file_recovery_contents";
fileWritten.size = archiveFileSize;
fileWritten.checksumType = checksumType;
fileWritten.checksumValue = checksumValue;
fileWritten.storageClassName = storageClass.name;
fileWritten.vid = vid;
fileWritten.fSeq = i;
fileWritten.blockId = i * 100;
fileWritten.compressedSize = compressedFileSize;
fileWritten.copyNb = copyNb;
const auto idAndFile = m.find(i);
ASSERT_FALSE(m.end() == idAndFile);
const common::dataStructures::ArchiveFile archiveFile = idAndFile->second;
ASSERT_EQ(fileWritten.archiveFileId, archiveFile.archiveFileID);
ASSERT_EQ(fileWritten.diskInstance, archiveFile.diskInstance);
ASSERT_EQ(fileWritten.diskFileId, archiveFile.diskFileId);
ASSERT_EQ(fileWritten.diskFilePath, archiveFile.diskFileInfo.path);
ASSERT_EQ(fileWritten.diskFileUser, archiveFile.diskFileInfo.owner);
ASSERT_EQ(fileWritten.diskFileGroup, archiveFile.diskFileInfo.group);
ASSERT_EQ(fileWritten.diskFileRecoveryBlob, archiveFile.diskFileInfo.recoveryBlob);
ASSERT_EQ(fileWritten.size, archiveFile.fileSize);
ASSERT_EQ(fileWritten.checksumType, archiveFile.checksumType);
ASSERT_EQ(fileWritten.checksumValue, archiveFile.checksumValue);
ASSERT_EQ(fileWritten.storageClassName, archiveFile.storageClass);
// There is only one tape copy because repack only want the tape file on a
// single tape
ASSERT_EQ(1, archiveFile.tapeFiles.size());
{
const auto it = archiveFile.tapeFiles.find(copyNb);
ASSERT_NE(archiveFile.tapeFiles.end(), it);
ASSERT_EQ(fileWritten.vid, it->second.vid);
ASSERT_EQ(fileWritten.fSeq, it->second.fSeq);
ASSERT_EQ(fileWritten.blockId, it->second.blockId);
ASSERT_EQ(fileWritten.compressedSize, it->second.compressedSize);
ASSERT_EQ(fileWritten.checksumType, it->second.checksumType);
ASSERT_EQ(fileWritten.checksumValue, it->second.checksumValue);
ASSERT_EQ(fileWritten.copyNb, it->second.copyNb);
ASSERT_EQ(fileWritten.copyNb, it->first);
}
}
}
for(uint64_t copyNb = 1; copyNb <= 2; copyNb++) {
const std::string vid = copyNb == 1 ? vid1 : vid2;
const uint64_t startFseq = 1;
const uint64_t maxNbFiles = nbArchiveFiles / 2;
auto archiveFiles = m_catalogue->getFilesForRepack(vid, startFseq, maxNbFiles);
auto m = archiveFileListToMap(archiveFiles);
ASSERT_EQ(nbArchiveFiles, m.size());
for(uint64_t i = nbArchiveFiles / 2 + 1; i <= nbArchiveFiles; i++) {
std::ostringstream diskFileId;
diskFileId << (12345677 + i);
std::ostringstream diskFilePath;
diskFilePath << "/public_dir/public_file_" << i;
catalogue::TapeFileWritten fileWritten;
fileWritten.archiveFileId = i;
fileWritten.diskInstance = storageClass.diskInstance;
fileWritten.diskFileId = diskFileId.str();
fileWritten.diskFilePath = diskFilePath.str();
fileWritten.diskFileUser = "public_disk_user";
fileWritten.diskFileGroup = "public_disk_group";
fileWritten.diskFileRecoveryBlob = "opaque_disk_file_recovery_contents";
fileWritten.size = archiveFileSize;
fileWritten.checksumType = checksumType;
fileWritten.checksumValue = checksumValue;
fileWritten.storageClassName = storageClass.name;
fileWritten.vid = vid;
fileWritten.fSeq = i;
fileWritten.blockId = i * 100;
fileWritten.compressedSize = compressedFileSize;
fileWritten.copyNb = copyNb;
const auto idAndFile = m.find(i);
ASSERT_FALSE(m.end() == idAndFile);
const common::dataStructures::ArchiveFile archiveFile = idAndFile->second;
ASSERT_EQ(fileWritten.archiveFileId, archiveFile.archiveFileID);
ASSERT_EQ(fileWritten.diskInstance, archiveFile.diskInstance);
ASSERT_EQ(fileWritten.diskFileId, archiveFile.diskFileId);
ASSERT_EQ(fileWritten.diskFilePath, archiveFile.diskFileInfo.path);
ASSERT_EQ(fileWritten.diskFileUser, archiveFile.diskFileInfo.owner);
ASSERT_EQ(fileWritten.diskFileGroup, archiveFile.diskFileInfo.group);
ASSERT_EQ(fileWritten.diskFileRecoveryBlob, archiveFile.diskFileInfo.recoveryBlob);
ASSERT_EQ(fileWritten.size, archiveFile.fileSize);
ASSERT_EQ(fileWritten.checksumType, archiveFile.checksumType);
ASSERT_EQ(fileWritten.checksumValue, archiveFile.checksumValue);
ASSERT_EQ(fileWritten.storageClassName, archiveFile.storageClass);
// There is only one tape copy because repack only want the tape file on a
// single tape
ASSERT_EQ(1, archiveFile.tapeFiles.size());
{
const auto it = archiveFile.tapeFiles.find(copyNb);
ASSERT_NE(archiveFile.tapeFiles.end(), it);
ASSERT_EQ(fileWritten.vid, it->second.vid);
ASSERT_EQ(fileWritten.fSeq, it->second.fSeq);
ASSERT_EQ(fileWritten.blockId, it->second.blockId);
ASSERT_EQ(fileWritten.compressedSize, it->second.compressedSize);
ASSERT_EQ(fileWritten.checksumType, it->second.checksumType);
ASSERT_EQ(fileWritten.checksumValue, it->second.checksumValue);
ASSERT_EQ(fileWritten.copyNb, it->second.copyNb);
ASSERT_EQ(fileWritten.copyNb, it->first);
}
}
}
{
catalogue::TapeFileSearchCriteria searchCriteria;
searchCriteria.archiveFileId = 10;
......
......@@ -73,6 +73,17 @@ protected:
std::map<uint64_t, cta::common::dataStructures::ArchiveFile> archiveFileItorToMap(
cta::catalogue::ArchiveFileItor &itor);
/**
* Creates a map from archive file ID to archive file from the specified
* list of archive files.
*
* @param listOfArchiveFiles The list of archive files from which the map
* shall be created.
* @return Map from archive file ID to archive file.
*/
std::map<uint64_t, cta::common::dataStructures::ArchiveFile> archiveFileListToMap(
const std::list<cta::common::dataStructures::ArchiveFile> &listOfArchiveFiles);
/**
* Creates a map from admin username to admin user from the specified list of
* admin users.
......
......@@ -57,7 +57,7 @@ public:
std::list<common::dataStructures::AdminUser> getAdminUsers() const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
common::dataStructures::ArchiveFile getArchiveFileById(const uint64_t id) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
ArchiveFileItor getArchiveFiles(const TapeFileSearchCriteria& searchCriteria) const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
std::list<common::dataStructures::TapeFile> getTapeFilesForRepack(const std::string &vid, const uint64_t startFSeq, const uint64_t maxNbFiles) const override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
std::list<common::dataStructures::ArchiveFile> getFilesForRepack(const std::string &vid, const uint64_t startFSeq, const uint64_t maxNbFiles) const override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
std::list<common::dataStructures::ArchiveRoute> getArchiveRoutes() const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
std::list<common::dataStructures::LogicalLibrary> getLogicalLibraries() const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
std::list<common::dataStructures::MountPolicy> getMountPolicies() const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
......
......@@ -4022,13 +4022,91 @@ ArchiveFileItor RdbmsCatalogue::getArchiveFiles(const TapeFileSearchCriteria &se
}
//------------------------------------------------------------------------------
// getTapeFilesForRepack
// getFilesForRepack
//------------------------------------------------------------------------------
std::list<common::dataStructures::TapeFile> RdbmsCatalogue::getTapeFilesForRepack(
std::list<common::dataStructures::ArchiveFile> RdbmsCatalogue::getFilesForRepack(
const std::string &vid,
const uint64_t startFSeq,
const uint64_t maxNbFiles) const {
return std::list<common::dataStructures::TapeFile>();
try {
std::string sql =
"SELECT "
"ARCHIVE_FILE.ARCHIVE_FILE_ID AS ARCHIVE_FILE_ID,"
"ARCHIVE_FILE.DISK_INSTANCE_NAME AS DISK_INSTANCE_NAME,"
"ARCHIVE_FILE.DISK_FILE_ID AS DISK_FILE_ID,"
"ARCHIVE_FILE.DISK_FILE_PATH AS DISK_FILE_PATH,"
"ARCHIVE_FILE.DISK_FILE_USER AS DISK_FILE_USER,"
"ARCHIVE_FILE.DISK_FILE_GROUP AS DISK_FILE_GROUP,"
"ARCHIVE_FILE.DISK_FILE_RECOVERY_BLOB AS DISK_FILE_RECOVERY_BLOB,"
"ARCHIVE_FILE.SIZE_IN_BYTES AS SIZE_IN_BYTES,"
"ARCHIVE_FILE.CHECKSUM_TYPE AS CHECKSUM_TYPE,"
"ARCHIVE_FILE.CHECKSUM_VALUE AS CHECKSUM_VALUE,"
"STORAGE_CLASS.STORAGE_CLASS_NAME AS STORAGE_CLASS_NAME,"
"ARCHIVE_FILE.CREATION_TIME AS ARCHIVE_FILE_CREATION_TIME,"
"ARCHIVE_FILE.RECONCILIATION_TIME AS RECONCILIATION_TIME,"
"TAPE_FILE.VID AS VID,"
"TAPE_FILE.FSEQ AS FSEQ,"
"TAPE_FILE.BLOCK_ID AS BLOCK_ID,"
"TAPE_FILE.COMPRESSED_SIZE_IN_BYTES AS COMPRESSED_SIZE_IN_BYTES,"
"TAPE_FILE.COPY_NB AS COPY_NB,"
"TAPE_FILE.CREATION_TIME AS TAPE_FILE_CREATION_TIME, "
"TAPE.TAPE_POOL_NAME AS TAPE_POOL_NAME "
"FROM "
"ARCHIVE_FILE "
"INNER JOIN STORAGE_CLASS ON "
"ARCHIVE_FILE.STORAGE_CLASS_ID = STORAGE_CLASS.STORAGE_CLASS_ID "
"INNER JOIN TAPE_FILE ON "
"ARCHIVE_FILE.ARCHIVE_FILE_ID = TAPE_FILE.ARCHIVE_FILE_ID "
"INNER JOIN TAPE ON "
"TAPE_FILE.VID = TAPE.VID "
"WHERE "
"TAPE_FILE.VID = :VID "
"ORDER BY FSEQ";
auto conn = m_connPool.getConn();
auto stmt = conn.createStmt(sql);
stmt.bindString(":VID", vid);
auto rset = stmt.executeQuery(rdbms::AutocommitMode::AUTOCOMMIT_OFF);
std::list<common::dataStructures::ArchiveFile> archiveFiles;
while(rset.next()) {
common::dataStructures::ArchiveFile archiveFile;
archiveFile.archiveFileID = rset.columnUint64("ARCHIVE_FILE_ID");
archiveFile.diskInstance = rset.columnString("DISK_INSTANCE_NAME");
archiveFile.diskFileId = rset.columnString("DISK_FILE_ID");
archiveFile.diskFileInfo.path = rset.columnString("DISK_FILE_PATH");
archiveFile.diskFileInfo.owner = rset.columnString("DISK_FILE_USER");
archiveFile.diskFileInfo.group = rset.columnString("DISK_FILE_GROUP");
archiveFile.diskFileInfo.recoveryBlob = rset.columnString("DISK_FILE_RECOVERY_BLOB");
archiveFile.fileSize = rset.columnUint64("SIZE_IN_BYTES");
archiveFile.checksumType = rset.columnString("CHECKSUM_TYPE");
archiveFile.checksumValue = rset.columnString("CHECKSUM_VALUE");
archiveFile.storageClass = rset.columnString("STORAGE_CLASS_NAME");
archiveFile.creationTime = rset.columnUint64("ARCHIVE_FILE_CREATION_TIME");
archiveFile.reconciliationTime = rset.columnUint64("RECONCILIATION_TIME");
common::dataStructures::TapeFile tapeFile;
tapeFile.vid = rset.columnString("VID");
tapeFile.fSeq = rset.columnUint64("FSEQ");
tapeFile.blockId = rset.columnUint64("BLOCK_ID");
tapeFile.compressedSize = rset.columnUint64("COMPRESSED_SIZE_IN_BYTES");
tapeFile.copyNb = rset.columnUint64("COPY_NB");
tapeFile.creationTime = rset.columnUint64("TAPE_FILE_CREATION_TIME");
tapeFile.checksumType = archiveFile.checksumType; // Duplicated for convenience
tapeFile.checksumValue = archiveFile.checksumValue; // Duplicated for convenience
archiveFile.tapeFiles[rset.columnUint64("COPY_NB")] = tapeFile;
archiveFiles.push_back(archiveFile);
}
return archiveFiles;
} catch(exception::UserError &) {
throw;
} catch(exception::Exception &ex) {
ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str());
throw;
}
}
//------------------------------------------------------------------------------
......
......@@ -496,15 +496,15 @@ public:
ArchiveFileItor getArchiveFiles(const TapeFileSearchCriteria &searchCriteria) const override;
/**
* Returns the specified tape files in file sequence order.
* Returns the specified files in tape file sequence order.
*
* @param vid The volume identifier of the tape.
* @param startFSeq The file sequence number of the first file. Please note
* that there might not be a file with this exact file sequence number.
* @param maxNbFiles The maximum number of files to be returned.
* @return The tape files.
* @return The specified files in tape file sequence order.
*/
std::list<common::dataStructures::TapeFile> getTapeFilesForRepack(
std::list<common::dataStructures::ArchiveFile> getFilesForRepack(
const std::string &vid,
const uint64_t startFSeq,
const uint64_t maxNbFiles) const override;
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment