From 5fa1952f8d2664ef608b1e9b1198d5790801b0f8 Mon Sep 17 00:00:00 2001 From: Steven Murray <steven.murray@cern.ch> Date: Fri, 27 May 2016 09:36:44 +0200 Subject: [PATCH] Added cta_catalogue_RdbmsCatalogueTest.prepareToRetrieveFile --- catalogue/Catalogue.hpp | 2 +- catalogue/RdbmsCatalogue.cpp | 82 ++-- catalogue/RdbmsCatalogue.hpp | 18 +- catalogue/RdbmsCatalogueTest.cpp | 374 +++++++++++++----- catalogue/TestingRdbmsCatalogue.hpp | 1 - .../RetrieveFileQueueCriteria.cpp | 2 +- .../RetrieveFileQueueCriteria.hpp | 13 +- scheduler/OStoreDB/OStoreDB.cpp | 3 +- scheduler/OStoreDB/OStoreDB.hpp | 3 +- scheduler/OStoreDB/OStoreDBFactory.hpp | 5 +- scheduler/Scheduler.cpp | 10 +- scheduler/SchedulerDatabase.hpp | 34 +- 12 files changed, 373 insertions(+), 174 deletions(-) diff --git a/catalogue/Catalogue.hpp b/catalogue/Catalogue.hpp index 2157c4853d..3ae18222a8 100644 --- a/catalogue/Catalogue.hpp +++ b/catalogue/Catalogue.hpp @@ -236,7 +236,7 @@ public: */ virtual common::dataStructures::RetrieveFileQueueCriteria prepareToRetrieveFile( const uint64_t archiveFileId, - common::dataStructures::UserIdentity &user) = 0; + const common::dataStructures::UserIdentity &user) = 0; virtual common::dataStructures::TapeCopyToPoolMap getTapeCopyToPoolMap(const std::string &storageClass) const = 0; diff --git a/catalogue/RdbmsCatalogue.cpp b/catalogue/RdbmsCatalogue.cpp index cb9a1f96be..d7b5aeee68 100644 --- a/catalogue/RdbmsCatalogue.cpp +++ b/catalogue/RdbmsCatalogue.cpp @@ -2162,12 +2162,55 @@ void RdbmsCatalogue::throwIfCommonEventDataMismatch(const common::dataStructures } //------------------------------------------------------------------------------ -// prepareToRetreiveFile +// prepareToRetrieveFile //------------------------------------------------------------------------------ common::dataStructures::RetrieveFileQueueCriteria RdbmsCatalogue::prepareToRetrieveFile( const uint64_t archiveFileId, - common::dataStructures::UserIdentity &user) { - throw exception::Exception(std::string(__FUNCTION__) + " not implemented"); + const common::dataStructures::UserIdentity &user) { + const std::map<uint64_t, common::dataStructures::TapeFile> tapeFiles = getTapeFiles(archiveFileId); + const common::dataStructures::MountPolicy mountPolicy = getMountPolicyForAUser(user); + + return common::dataStructures::RetrieveFileQueueCriteria(tapeFiles, mountPolicy); +} + +//------------------------------------------------------------------------------ +// getTapeFiles +//------------------------------------------------------------------------------ +std::map<uint64_t, common::dataStructures::TapeFile> RdbmsCatalogue::getTapeFiles(const uint64_t archiveFileId) const { + try { + std::map<uint64_t, common::dataStructures::TapeFile> tapeFiles; + + const char *const sql = + "SELECT " + "TAPE_FILE.VID AS VID," + "TAPE_FILE.FSEQ AS FSEQ," + "TAPE_FILE.BLOCK_ID AS BLOCK_ID," + "TAPE_FILE.COMPRESSED_SIZE AS COMPRESSED_SIZE," + "TAPE_FILE.COPY_NB AS COPY_NB," + "TAPE_FILE.CREATION_TIME AS TAPE_FILE_CREATION_TIME " + "FROM " + "TAPE_FILE " + "WHERE " + "TAPE_FILE.ARCHIVE_FILE_ID = :ARCHIVE_FILE_ID;"; + std::unique_ptr<DbStmt> stmt(m_conn.createStmt(sql)); + stmt->bindUint64(":ARCHIVE_FILE_ID", archiveFileId); + std::unique_ptr<DbRset> rset(stmt->executeQuery()); + std::unique_ptr<common::dataStructures::ArchiveFile> archiveFile; + while (rset->next()) { + common::dataStructures::TapeFile tapeFile; + tapeFile.vid = rset->columnText("VID"); + tapeFile.fSeq = rset->columnUint64("FSEQ"); + tapeFile.blockId = rset->columnUint64("BLOCK_ID"); + tapeFile.compressedSize = rset->columnUint64("COMPRESSED_SIZE"); + tapeFile.copyNb = rset->columnUint64("COPY_NB"); + tapeFile.creationTime = rset->columnUint64("TAPE_FILE_CREATION_TIME"); + tapeFiles[rset->columnUint64("COPY_NB")] = tapeFile; + } + + return tapeFiles; + } catch(exception::Exception &ex) { + throw exception::Exception(std::string(__FUNCTION__) + " failed: " + ex.getMessage().str()); + } } //------------------------------------------------------------------------------ @@ -2280,38 +2323,6 @@ void RdbmsCatalogue::createTapeFile(const common::dataStructures::TapeFile &tape stmt->executeNonQuery(); } -//------------------------------------------------------------------------------ -// getTapeFiles -//------------------------------------------------------------------------------ -std::list<common::dataStructures::TapeFile> RdbmsCatalogue::getTapeFiles() const { - std::list<common::dataStructures::TapeFile> files; - const char *const sql = - "SELECT " - "VID AS VID," - "FSEQ AS FSEQ," - "BLOCK_ID AS BLOCK_ID," - "COMPRESSED_SIZE AS COMPRESSED_SIZE," - "COPY_NB AS COPY_NB," - "CREATION_TIME AS CREATION_TIME " - "FROM TAPE_FILE;"; - std::unique_ptr<DbStmt> stmt(m_conn.createStmt(sql)); - std::unique_ptr<DbRset> rset(stmt->executeQuery()); - while(rset->next()) { - common::dataStructures::TapeFile file; - - file.vid = rset->columnText("VID"); - file.fSeq = rset->columnUint64("FSEQ"); - file.blockId = rset->columnUint64("BLOCK_ID"); - file.compressedSize = rset->columnUint64("COMPRESSED_SIZE"); - file.copyNb = rset->columnUint64("COPY_NB"); - file.creationTime = rset->columnUint64("CREATION_TIME"); - - files.push_back(file); - } - - return files; -} - //------------------------------------------------------------------------------ // setTapeLastFseq //------------------------------------------------------------------------------ @@ -2421,6 +2432,7 @@ std::unique_ptr<common::dataStructures::ArchiveFile> RdbmsCatalogue::getArchiveF tapeFile.fSeq = rset->columnUint64("FSEQ"); tapeFile.blockId = rset->columnUint64("BLOCK_ID"); tapeFile.compressedSize = rset->columnUint64("COMPRESSED_SIZE"); + tapeFile.copyNb = rset->columnUint64("COPY_NB"); tapeFile.creationTime = rset->columnUint64("TAPE_FILE_CREATION_TIME"); archiveFile->tapeFiles[rset->columnUint64("COPY_NB")] = tapeFile; } diff --git a/catalogue/RdbmsCatalogue.hpp b/catalogue/RdbmsCatalogue.hpp index 3827904674..e2a678464d 100644 --- a/catalogue/RdbmsCatalogue.hpp +++ b/catalogue/RdbmsCatalogue.hpp @@ -213,7 +213,7 @@ public: */ virtual common::dataStructures::RetrieveFileQueueCriteria prepareToRetrieveFile( const uint64_t archiveFileId, - common::dataStructures::UserIdentity &user); + const common::dataStructures::UserIdentity &user); virtual common::dataStructures::TapeCopyToPoolMap getTapeCopyToPoolMap(const std::string &storageClass) const; @@ -320,13 +320,6 @@ protected: */ void createTapeFile(const common::dataStructures::TapeFile &tapeFile, const uint64_t archiveFileId); - /** - * Returns the list of all the tape files in the catalogue. - * - * @return The list of all the tape files in the catalogue. - */ - std::list<common::dataStructures::TapeFile> getTapeFiles() const; - /** * Sets the last FSeq of the specified tape to the specified value. * @@ -361,6 +354,15 @@ protected: */ void throwIfCommonEventDataMismatch(const common::dataStructures::ArchiveFile &expected, const TapeFileWritten &actual) const; + + /** + * Returns the tape files for the specified archive file. + * + * @param archiveFileId The unique identifier of the archive file. + * @return The tape files as a map from tape copy number to tape file. + */ + std::map<uint64_t, common::dataStructures::TapeFile>getTapeFiles(const uint64_t archiveFileId) const; + }; // class RdbmsCatalogue } // namespace catalogue diff --git a/catalogue/RdbmsCatalogueTest.cpp b/catalogue/RdbmsCatalogueTest.cpp index bb9aace6a5..8d9da67927 100644 --- a/catalogue/RdbmsCatalogueTest.cpp +++ b/catalogue/RdbmsCatalogueTest.cpp @@ -876,8 +876,7 @@ TEST_F(cta_catalogue_RdbmsCatalogueTest, insertArchiveFile) { const std::string storageClassName = "storage_class"; const uint64_t nbCopies = 2; - m_catalogue->createStorageClass(m_cliSI, storageClassName, nbCopies, - "create storage class"); + m_catalogue->createStorageClass(m_cliSI, storageClassName, nbCopies, "create storage class"); catalogue::ArchiveFileRow row; row.archiveFileId = 1234; @@ -977,7 +976,7 @@ TEST_F(cta_catalogue_RdbmsCatalogueTest, prepareForNewFile) { ASSERT_EQ(userComment, user.comment); ASSERT_EQ(m_cliSI.user, user.creationLog.user); ASSERT_EQ(m_cliSI.host, user.creationLog.host); - ASSERT_EQ(user.creationLog, user.lastModificationLog); + ASSERT_EQ(user.creationLog, user.lastModificationLog); const common::dataStructures::MountPolicy policy = m_catalogue->getMountPolicyForAUser(userIdentity); @@ -994,7 +993,7 @@ TEST_F(cta_catalogue_RdbmsCatalogueTest, prepareForNewFile) { const uint64_t nbCopies = 2; m_catalogue->createStorageClass(m_cliSI, storageClassName, nbCopies, "create storage class"); - + const std::string tapePoolName = "tape_pool"; const uint64_t nbPartialTapes = 2; const bool is_encrypted = true; @@ -1005,12 +1004,12 @@ TEST_F(cta_catalogue_RdbmsCatalogueTest, prepareForNewFile) { const std::string archiveRouteComment = "create archive route"; m_catalogue->createArchiveRoute(m_cliSI, storageClassName, copyNb, tapePoolName, archiveRouteComment); - + const std::list<common::dataStructures::ArchiveRoute> routes = m_catalogue->getArchiveRoutes(); - + ASSERT_EQ(1, routes.size()); - + const common::dataStructures::ArchiveRoute route = routes.front(); ASSERT_EQ(storageClassName, route.storageClassName); ASSERT_EQ(copyNb, route.copyNb); @@ -1021,7 +1020,7 @@ TEST_F(cta_catalogue_RdbmsCatalogueTest, prepareForNewFile) { ASSERT_EQ(m_cliSI.user.name, creationLog.user.name); ASSERT_EQ(m_cliSI.user.group, creationLog.user.group); ASSERT_EQ(m_cliSI.host, creationLog.host); - + const common::dataStructures::EntryLog lastModificationLog = route.lastModificationLog; ASSERT_EQ(creationLog, lastModificationLog); @@ -1045,6 +1044,170 @@ TEST_F(cta_catalogue_RdbmsCatalogueTest, prepareForNewFile) { ASSERT_EQ(maxDrivesAllowed, queueCriteria.mountPolicy.maxDrivesAllowed); } +TEST_F(cta_catalogue_RdbmsCatalogueTest, prepareToRetrieveFile) { + using namespace cta; + + const uint64_t archiveFileId = 1234; + + ASSERT_TRUE(m_catalogue->getArchiveFiles("", "", "", "", "", "", "", "", "").empty()); + ASSERT_TRUE(NULL == m_catalogue->getArchiveFile(archiveFileId).get()); + + const std::string storageClassName = "storage_class"; + const uint64_t nbCopies = 2; + m_catalogue->createStorageClass(m_cliSI, storageClassName, nbCopies, "create storage class"); + + const uint64_t archiveFileSize = 1; + + catalogue::TapeFileWritten file1Written; + file1Written.archiveFileId = archiveFileId; + file1Written.diskInstance = "PUBLIC"; + file1Written.diskFileId = "5678"; + file1Written.diskFilePath = "/public_dir/public_file"; + file1Written.diskFileUser = "public_disk_user"; + file1Written.diskFileGroup = "public_disk_group"; + file1Written.diskFileRecoveryBlob = "opaque_disk_file_recovery_contents"; + file1Written.size = archiveFileSize; + file1Written.storageClassName = storageClassName; + file1Written.vid = "VID123"; + file1Written.fSeq = 1; + file1Written.blockId = 4321; + file1Written.compressedSize = 1; + file1Written.copyNb = 1; + m_catalogue->fileWrittenToTape(file1Written); + + { + std::unique_ptr<common::dataStructures::ArchiveFile> archivedFile = m_catalogue->getArchiveFile(archiveFileId); + ASSERT_TRUE(NULL != archivedFile.get()); + + ASSERT_EQ(file1Written.archiveFileId, archivedFile->archiveFileID); + ASSERT_EQ(file1Written.diskFileId, archivedFile->diskFileID); + ASSERT_EQ(file1Written.size, archivedFile->fileSize); + ASSERT_EQ(file1Written.storageClassName, archivedFile->storageClass); + + ASSERT_EQ(file1Written.diskInstance, archivedFile->diskInstance); + ASSERT_EQ(file1Written.diskFilePath, archivedFile->drData.drPath); + ASSERT_EQ(file1Written.diskFileUser, archivedFile->drData.drOwner); + ASSERT_EQ(file1Written.diskFileGroup, archivedFile->drData.drGroup); + ASSERT_EQ(file1Written.diskFileRecoveryBlob, archivedFile->drData.drBlob); + + ASSERT_EQ(1, archivedFile->tapeFiles.size()); + auto copyNbToTapeFile1Itor = archivedFile->tapeFiles.find(1); + ASSERT_FALSE(copyNbToTapeFile1Itor == archivedFile->tapeFiles.end()); + const common::dataStructures::TapeFile &tapeFile1 = copyNbToTapeFile1Itor->second; + ASSERT_EQ(file1Written.vid, tapeFile1.vid); + ASSERT_EQ(file1Written.fSeq, tapeFile1.fSeq); + ASSERT_EQ(file1Written.blockId, tapeFile1.blockId); + ASSERT_EQ(file1Written.compressedSize, tapeFile1.compressedSize); + ASSERT_EQ(file1Written.copyNb, tapeFile1.copyNb); + } + + catalogue::TapeFileWritten file2Written; + file2Written.archiveFileId = file1Written.archiveFileId; + file2Written.diskInstance = file1Written.diskInstance; + file2Written.diskFileId = file1Written.diskFileId; + file2Written.diskFilePath = file1Written.diskFilePath; + file2Written.diskFileUser = file1Written.diskFileUser; + file2Written.diskFileGroup = file1Written.diskFileGroup; + file2Written.diskFileRecoveryBlob = file1Written.diskFileRecoveryBlob; + file2Written.size = archiveFileSize; + file2Written.storageClassName = storageClassName; + file2Written.vid = "VID123"; + file2Written.fSeq = 2; + file2Written.blockId = 4331; + file2Written.compressedSize = 1; + file2Written.copyNb = 2; + m_catalogue->fileWrittenToTape(file2Written); + + { + std::unique_ptr<common::dataStructures::ArchiveFile> archivedFile = m_catalogue->getArchiveFile(archiveFileId); + ASSERT_TRUE(NULL != archivedFile.get()); + + ASSERT_EQ(file2Written.archiveFileId, archivedFile->archiveFileID); + ASSERT_EQ(file2Written.diskFileId, archivedFile->diskFileID); + ASSERT_EQ(file2Written.size, archivedFile->fileSize); + ASSERT_EQ(file2Written.storageClassName, archivedFile->storageClass); + + ASSERT_EQ(file2Written.diskInstance, archivedFile->diskInstance); + ASSERT_EQ(file2Written.diskFilePath, archivedFile->drData.drPath); + ASSERT_EQ(file2Written.diskFileUser, archivedFile->drData.drOwner); + ASSERT_EQ(file2Written.diskFileGroup, archivedFile->drData.drGroup); + ASSERT_EQ(file2Written.diskFileRecoveryBlob, archivedFile->drData.drBlob); + + ASSERT_EQ(2, archivedFile->tapeFiles.size()); + + auto copyNbToTapeFile1Itor = archivedFile->tapeFiles.find(1); + ASSERT_FALSE(copyNbToTapeFile1Itor == archivedFile->tapeFiles.end()); + const common::dataStructures::TapeFile &tapeFile1 = copyNbToTapeFile1Itor->second; + ASSERT_EQ(file1Written.vid, tapeFile1.vid); + ASSERT_EQ(file1Written.fSeq, tapeFile1.fSeq); + ASSERT_EQ(file1Written.blockId, tapeFile1.blockId); + ASSERT_EQ(file1Written.compressedSize, tapeFile1.compressedSize); + + auto copyNbToTapeFile2Itor = archivedFile->tapeFiles.find(2); + ASSERT_FALSE(copyNbToTapeFile2Itor == archivedFile->tapeFiles.end()); + const common::dataStructures::TapeFile &tapeFile2 = copyNbToTapeFile2Itor->second; + ASSERT_EQ(file2Written.vid, tapeFile2.vid); + ASSERT_EQ(file2Written.fSeq, tapeFile2.fSeq); + ASSERT_EQ(file2Written.blockId, tapeFile2.blockId); + ASSERT_EQ(file2Written.compressedSize, tapeFile2.compressedSize); + } + + const std::string mountPolicyName = "mount_group"; + const uint64_t archivePriority = 1; + const uint64_t minArchiveRequestAge = 2; + const uint64_t retrievePriority = 3; + const uint64_t minRetrieveRequestAge = 4; + const uint64_t maxDrivesAllowed = 5; + + m_catalogue->createMountPolicy( + m_cliSI, + mountPolicyName, + archivePriority, + minArchiveRequestAge, + retrievePriority, + minRetrieveRequestAge, + maxDrivesAllowed, + "create mount group"); + + const std::string userComment = "create user"; + const std::string userName = "user_name"; + const std::string group = "group"; + cta::common::dataStructures::UserIdentity userIdentity; + userIdentity.name=userName; + userIdentity.group=group; + m_catalogue->createRequester(m_cliSI, userIdentity, mountPolicyName, userComment); + + std::list<common::dataStructures::Requester> users; + users = m_catalogue->getRequesters(); + ASSERT_EQ(1, users.size()); + + const common::dataStructures::Requester user = users.front(); + + ASSERT_EQ(userName, user.name); + ASSERT_EQ(mountPolicyName, user.mountPolicy); + ASSERT_EQ(userComment, user.comment); + ASSERT_EQ(m_cliSI.user, user.creationLog.user); + ASSERT_EQ(m_cliSI.host, user.creationLog.host); + ASSERT_EQ(user.creationLog, user.lastModificationLog); + + const common::dataStructures::MountPolicy policy = + m_catalogue->getMountPolicyForAUser(userIdentity); + + ASSERT_EQ(archivePriority, policy.archivePriority); + ASSERT_EQ(minArchiveRequestAge, policy.archiveMinRequestAge); + ASSERT_EQ(maxDrivesAllowed, policy.maxDrivesAllowed); + ASSERT_EQ(retrievePriority, policy.retrievePriority); + ASSERT_EQ(minRetrieveRequestAge, policy.retrieveMinRequestAge); + + const common::dataStructures::RetrieveFileQueueCriteria queueCriteria = + m_catalogue->prepareToRetrieveFile(archiveFileId, userIdentity); + + ASSERT_EQ(2, queueCriteria.tapeFiles.size()); + ASSERT_EQ(archivePriority, queueCriteria.mountPolicy.archivePriority); + ASSERT_EQ(minArchiveRequestAge, queueCriteria.mountPolicy.archiveMinRequestAge); + ASSERT_EQ(maxDrivesAllowed, queueCriteria.mountPolicy.maxDrivesAllowed); +} + TEST_F(cta_catalogue_RdbmsCatalogueTest, createTapeFile_2_files) { using namespace cta; @@ -1093,8 +1256,6 @@ TEST_F(cta_catalogue_RdbmsCatalogueTest, createTapeFile_2_files) { ASSERT_TRUE(archiveFile.tapeFiles.empty()); } - ASSERT_TRUE(m_catalogue->getTapeFiles().empty()); - common::dataStructures::TapeFile tapeFile1; tapeFile1.vid = "VID1"; tapeFile1.fSeq = 5678; @@ -1125,27 +1286,16 @@ TEST_F(cta_catalogue_RdbmsCatalogueTest, createTapeFile_2_files) { ASSERT_EQ(1, archiveFile.tapeFiles.size()); { - auto copyNbToTapeFileItor = archiveFile.tapeFiles.find(1); - ASSERT_EQ(1, copyNbToTapeFileItor->first); - ASSERT_EQ(tapeFile1.vid, copyNbToTapeFileItor->second.vid); - ASSERT_EQ(tapeFile1.fSeq, copyNbToTapeFileItor->second.fSeq); - ASSERT_EQ(tapeFile1.blockId, copyNbToTapeFileItor->second.blockId); - ASSERT_EQ(tapeFile1.compressedSize, copyNbToTapeFileItor->second.compressedSize); - ASSERT_EQ(tapeFile1.copyNb, copyNbToTapeFileItor->second.copyNb); + auto copyNbToTapeFile1Itor = archiveFile.tapeFiles.find(1); + ASSERT_EQ(1, copyNbToTapeFile1Itor->first); + ASSERT_EQ(tapeFile1.vid, copyNbToTapeFile1Itor->second.vid); + ASSERT_EQ(tapeFile1.fSeq, copyNbToTapeFile1Itor->second.fSeq); + ASSERT_EQ(tapeFile1.blockId, copyNbToTapeFile1Itor->second.blockId); + ASSERT_EQ(tapeFile1.compressedSize, copyNbToTapeFile1Itor->second.compressedSize); + ASSERT_EQ(tapeFile1.copyNb, copyNbToTapeFile1Itor->second.copyNb); } } - { - const std::list<common::dataStructures::TapeFile> tapeFiles = m_catalogue->getTapeFiles(); - - ASSERT_EQ(1, tapeFiles.size()); - ASSERT_EQ(tapeFile1.vid, tapeFiles.front().vid); - ASSERT_EQ(tapeFile1.fSeq, tapeFiles.front().fSeq); - ASSERT_EQ(tapeFile1.blockId, tapeFiles.front().blockId); - ASSERT_EQ(tapeFile1.compressedSize, tapeFiles.front().compressedSize); - ASSERT_EQ(tapeFile1.copyNb, tapeFiles.front().copyNb); - } - common::dataStructures::TapeFile tapeFile2; tapeFile2.vid = "VID2"; tapeFile2.fSeq = 3456; @@ -1176,31 +1326,25 @@ TEST_F(cta_catalogue_RdbmsCatalogueTest, createTapeFile_2_files) { ASSERT_EQ(2, archiveFile.tapeFiles.size()); { - auto copyNbToTapeFileItor = archiveFile.tapeFiles.find(1); - ASSERT_EQ(1, copyNbToTapeFileItor->first); - ASSERT_EQ(tapeFile1.vid, copyNbToTapeFileItor->second.vid); - ASSERT_EQ(tapeFile1.fSeq, copyNbToTapeFileItor->second.fSeq); - ASSERT_EQ(tapeFile1.blockId, copyNbToTapeFileItor->second.blockId); - ASSERT_EQ(tapeFile1.compressedSize, copyNbToTapeFileItor->second.compressedSize); - ASSERT_EQ(tapeFile1.copyNb, copyNbToTapeFileItor->second.copyNb); + auto copyNbToTapeFile1Itor = archiveFile.tapeFiles.find(1); + ASSERT_EQ(1, copyNbToTapeFile1Itor->first); + ASSERT_EQ(tapeFile1.vid, copyNbToTapeFile1Itor->second.vid); + ASSERT_EQ(tapeFile1.fSeq, copyNbToTapeFile1Itor->second.fSeq); + ASSERT_EQ(tapeFile1.blockId, copyNbToTapeFile1Itor->second.blockId); + ASSERT_EQ(tapeFile1.compressedSize, copyNbToTapeFile1Itor->second.compressedSize); + ASSERT_EQ(tapeFile1.copyNb, copyNbToTapeFile1Itor->second.copyNb); } { - auto copyNbToTapeFileItor = archiveFile.tapeFiles.find(2); - ASSERT_EQ(2, copyNbToTapeFileItor->first); - ASSERT_EQ(tapeFile2.vid, copyNbToTapeFileItor->second.vid); - ASSERT_EQ(tapeFile2.fSeq, copyNbToTapeFileItor->second.fSeq); - ASSERT_EQ(tapeFile2.blockId, copyNbToTapeFileItor->second.blockId); - ASSERT_EQ(tapeFile2.compressedSize, copyNbToTapeFileItor->second.compressedSize); - ASSERT_EQ(tapeFile2.copyNb, copyNbToTapeFileItor->second.copyNb); + auto copyNbToTapeFile1Itor = archiveFile.tapeFiles.find(2); + ASSERT_EQ(2, copyNbToTapeFile1Itor->first); + ASSERT_EQ(tapeFile2.vid, copyNbToTapeFile1Itor->second.vid); + ASSERT_EQ(tapeFile2.fSeq, copyNbToTapeFile1Itor->second.fSeq); + ASSERT_EQ(tapeFile2.blockId, copyNbToTapeFile1Itor->second.blockId); + ASSERT_EQ(tapeFile2.compressedSize, copyNbToTapeFile1Itor->second.compressedSize); + ASSERT_EQ(tapeFile2.copyNb, copyNbToTapeFile1Itor->second.copyNb); } } - - { - const std::list<common::dataStructures::TapeFile> tapeFiles = m_catalogue->getTapeFiles(); - - ASSERT_EQ(2, tapeFiles.size()); - } } TEST_F(cta_catalogue_RdbmsCatalogueTest, getTapeLastFseq_no_such_tape) { @@ -1336,8 +1480,6 @@ TEST_F(cta_catalogue_RdbmsCatalogueTest, getArchiveFile) { ASSERT_TRUE(archiveFile.tapeFiles.empty()); } - ASSERT_TRUE(m_catalogue->getTapeFiles().empty()); - { std::unique_ptr<common::dataStructures::ArchiveFile> archiveFile = m_catalogue->getArchiveFile(archiveFileId); ASSERT_TRUE(NULL != archiveFile.get()); @@ -1355,8 +1497,6 @@ TEST_F(cta_catalogue_RdbmsCatalogueTest, getArchiveFile) { ASSERT_TRUE(archiveFile->tapeFiles.empty()); } - - ASSERT_TRUE(m_catalogue->getTapeFiles().empty()); } TEST_F(cta_catalogue_RdbmsCatalogueTest, fileWrittenToTape_2_tape_files) { @@ -1392,25 +1532,31 @@ TEST_F(cta_catalogue_RdbmsCatalogueTest, fileWrittenToTape_2_tape_files) { m_catalogue->fileWrittenToTape(file1Written); { - std::unique_ptr<common::dataStructures::ArchiveFile> retrievedFile = m_catalogue->getArchiveFile(archiveFileId); - ASSERT_TRUE(NULL != retrievedFile.get()); - - ASSERT_EQ(file1Written.archiveFileId, retrievedFile->archiveFileID); - ASSERT_EQ(file1Written.diskFileId, retrievedFile->diskFileID); - ASSERT_EQ(file1Written.size, retrievedFile->fileSize); - ASSERT_EQ(file1Written.storageClassName, retrievedFile->storageClass); - - ASSERT_EQ(file1Written.diskInstance, retrievedFile->diskInstance); - ASSERT_EQ(file1Written.diskFilePath, retrievedFile->drData.drPath); - ASSERT_EQ(file1Written.diskFileUser, retrievedFile->drData.drOwner); - ASSERT_EQ(file1Written.diskFileGroup, retrievedFile->drData.drGroup); - ASSERT_EQ(file1Written.diskFileRecoveryBlob, retrievedFile->drData.drBlob); - - ASSERT_EQ(1, retrievedFile->tapeFiles.size()); + std::unique_ptr<common::dataStructures::ArchiveFile> archivedFile = m_catalogue->getArchiveFile(archiveFileId); + ASSERT_TRUE(NULL != archivedFile.get()); + + ASSERT_EQ(file1Written.archiveFileId, archivedFile->archiveFileID); + ASSERT_EQ(file1Written.diskFileId, archivedFile->diskFileID); + ASSERT_EQ(file1Written.size, archivedFile->fileSize); + ASSERT_EQ(file1Written.storageClassName, archivedFile->storageClass); + + ASSERT_EQ(file1Written.diskInstance, archivedFile->diskInstance); + ASSERT_EQ(file1Written.diskFilePath, archivedFile->drData.drPath); + ASSERT_EQ(file1Written.diskFileUser, archivedFile->drData.drOwner); + ASSERT_EQ(file1Written.diskFileGroup, archivedFile->drData.drGroup); + ASSERT_EQ(file1Written.diskFileRecoveryBlob, archivedFile->drData.drBlob); + + ASSERT_EQ(1, archivedFile->tapeFiles.size()); + auto copyNbToTapeFile1Itor = archivedFile->tapeFiles.find(1); + ASSERT_FALSE(copyNbToTapeFile1Itor == archivedFile->tapeFiles.end()); + const common::dataStructures::TapeFile &tapeFile1 = copyNbToTapeFile1Itor->second; + ASSERT_EQ(file1Written.vid, tapeFile1.vid); + ASSERT_EQ(file1Written.fSeq, tapeFile1.fSeq); + ASSERT_EQ(file1Written.blockId, tapeFile1.blockId); + ASSERT_EQ(file1Written.compressedSize, tapeFile1.compressedSize); + ASSERT_EQ(file1Written.copyNb, tapeFile1.copyNb); } - ASSERT_EQ(1, m_catalogue->getTapeFiles().size()); - catalogue::TapeFileWritten file2Written; file2Written.archiveFileId = file1Written.archiveFileId; file2Written.diskInstance = file1Written.diskInstance; @@ -1429,24 +1575,40 @@ TEST_F(cta_catalogue_RdbmsCatalogueTest, fileWrittenToTape_2_tape_files) { m_catalogue->fileWrittenToTape(file2Written); { - std::unique_ptr<common::dataStructures::ArchiveFile> retrievedFile = m_catalogue->getArchiveFile(archiveFileId); - ASSERT_TRUE(NULL != retrievedFile.get()); - - ASSERT_EQ(file2Written.archiveFileId, retrievedFile->archiveFileID); - ASSERT_EQ(file2Written.diskFileId, retrievedFile->diskFileID); - ASSERT_EQ(file2Written.size, retrievedFile->fileSize); - ASSERT_EQ(file2Written.storageClassName, retrievedFile->storageClass); - - ASSERT_EQ(file2Written.diskInstance, retrievedFile->diskInstance); - ASSERT_EQ(file2Written.diskFilePath, retrievedFile->drData.drPath); - ASSERT_EQ(file2Written.diskFileUser, retrievedFile->drData.drOwner); - ASSERT_EQ(file2Written.diskFileGroup, retrievedFile->drData.drGroup); - ASSERT_EQ(file2Written.diskFileRecoveryBlob, retrievedFile->drData.drBlob); - - ASSERT_EQ(2, retrievedFile->tapeFiles.size()); + std::unique_ptr<common::dataStructures::ArchiveFile> archivedFile = m_catalogue->getArchiveFile(archiveFileId); + ASSERT_TRUE(NULL != archivedFile.get()); + + ASSERT_EQ(file2Written.archiveFileId, archivedFile->archiveFileID); + ASSERT_EQ(file2Written.diskFileId, archivedFile->diskFileID); + ASSERT_EQ(file2Written.size, archivedFile->fileSize); + ASSERT_EQ(file2Written.storageClassName, archivedFile->storageClass); + + ASSERT_EQ(file2Written.diskInstance, archivedFile->diskInstance); + ASSERT_EQ(file2Written.diskFilePath, archivedFile->drData.drPath); + ASSERT_EQ(file2Written.diskFileUser, archivedFile->drData.drOwner); + ASSERT_EQ(file2Written.diskFileGroup, archivedFile->drData.drGroup); + ASSERT_EQ(file2Written.diskFileRecoveryBlob, archivedFile->drData.drBlob); + + ASSERT_EQ(2, archivedFile->tapeFiles.size()); + + auto copyNbToTapeFile1Itor = archivedFile->tapeFiles.find(1); + ASSERT_FALSE(copyNbToTapeFile1Itor == archivedFile->tapeFiles.end()); + const common::dataStructures::TapeFile &tapeFile1 = copyNbToTapeFile1Itor->second; + ASSERT_EQ(file1Written.vid, tapeFile1.vid); + ASSERT_EQ(file1Written.fSeq, tapeFile1.fSeq); + ASSERT_EQ(file1Written.blockId, tapeFile1.blockId); + ASSERT_EQ(file1Written.compressedSize, tapeFile1.compressedSize); + ASSERT_EQ(file1Written.copyNb, tapeFile1.copyNb); + + auto copyNbToTapeFile2Itor = archivedFile->tapeFiles.find(2); + ASSERT_FALSE(copyNbToTapeFile2Itor == archivedFile->tapeFiles.end()); + const common::dataStructures::TapeFile &tapeFile2 = copyNbToTapeFile2Itor->second; + ASSERT_EQ(file2Written.vid, tapeFile2.vid); + ASSERT_EQ(file2Written.fSeq, tapeFile2.fSeq); + ASSERT_EQ(file2Written.blockId, tapeFile2.blockId); + ASSERT_EQ(file2Written.compressedSize, tapeFile2.compressedSize); + ASSERT_EQ(file2Written.copyNb, tapeFile2.copyNb); } - - ASSERT_EQ(2, m_catalogue->getTapeFiles().size()); } TEST_F(cta_catalogue_RdbmsCatalogueTest, fileWrittenToTape_2_tape_files_corrupted_diskFilePath) { @@ -1482,25 +1644,31 @@ TEST_F(cta_catalogue_RdbmsCatalogueTest, fileWrittenToTape_2_tape_files_corrupte m_catalogue->fileWrittenToTape(file1Written); { - std::unique_ptr<common::dataStructures::ArchiveFile> retrievedFile = m_catalogue->getArchiveFile(archiveFileId); - ASSERT_TRUE(NULL != retrievedFile.get()); - - ASSERT_EQ(file1Written.archiveFileId, retrievedFile->archiveFileID); - ASSERT_EQ(file1Written.diskFileId, retrievedFile->diskFileID); - ASSERT_EQ(file1Written.size, retrievedFile->fileSize); - ASSERT_EQ(file1Written.storageClassName, retrievedFile->storageClass); - - ASSERT_EQ(file1Written.diskInstance, retrievedFile->diskInstance); - ASSERT_EQ(file1Written.diskFilePath, retrievedFile->drData.drPath); - ASSERT_EQ(file1Written.diskFileUser, retrievedFile->drData.drOwner); - ASSERT_EQ(file1Written.diskFileGroup, retrievedFile->drData.drGroup); - ASSERT_EQ(file1Written.diskFileRecoveryBlob, retrievedFile->drData.drBlob); - - ASSERT_EQ(1, retrievedFile->tapeFiles.size()); + std::unique_ptr<common::dataStructures::ArchiveFile> archivedFile = m_catalogue->getArchiveFile(archiveFileId); + ASSERT_TRUE(NULL != archivedFile.get()); + + ASSERT_EQ(file1Written.archiveFileId, archivedFile->archiveFileID); + ASSERT_EQ(file1Written.diskFileId, archivedFile->diskFileID); + ASSERT_EQ(file1Written.size, archivedFile->fileSize); + ASSERT_EQ(file1Written.storageClassName, archivedFile->storageClass); + + ASSERT_EQ(file1Written.diskInstance, archivedFile->diskInstance); + ASSERT_EQ(file1Written.diskFilePath, archivedFile->drData.drPath); + ASSERT_EQ(file1Written.diskFileUser, archivedFile->drData.drOwner); + ASSERT_EQ(file1Written.diskFileGroup, archivedFile->drData.drGroup); + ASSERT_EQ(file1Written.diskFileRecoveryBlob, archivedFile->drData.drBlob); + + ASSERT_EQ(1, archivedFile->tapeFiles.size()); + auto copyNbToTapeFile1Itor = archivedFile->tapeFiles.find(1); + ASSERT_FALSE(copyNbToTapeFile1Itor == archivedFile->tapeFiles.end()); + const common::dataStructures::TapeFile &tapeFile1 = copyNbToTapeFile1Itor->second; + ASSERT_EQ(file1Written.vid, tapeFile1.vid); + ASSERT_EQ(file1Written.fSeq, tapeFile1.fSeq); + ASSERT_EQ(file1Written.blockId, tapeFile1.blockId); + ASSERT_EQ(file1Written.compressedSize, tapeFile1.compressedSize); + ASSERT_EQ(file1Written.copyNb, tapeFile1.copyNb); } - ASSERT_EQ(1, m_catalogue->getTapeFiles().size()); - catalogue::TapeFileWritten file2Written; file2Written.archiveFileId = file1Written.archiveFileId; file2Written.diskInstance = file1Written.diskInstance; diff --git a/catalogue/TestingRdbmsCatalogue.hpp b/catalogue/TestingRdbmsCatalogue.hpp index 223eff11e7..9ebd7f4fd4 100644 --- a/catalogue/TestingRdbmsCatalogue.hpp +++ b/catalogue/TestingRdbmsCatalogue.hpp @@ -47,7 +47,6 @@ public: using RdbmsCatalogue::insertArchiveFile; using RdbmsCatalogue::createTapeFile; using RdbmsCatalogue::getArchiveFile; - using RdbmsCatalogue::getTapeFiles; using RdbmsCatalogue::getTapeLastFSeq; using RdbmsCatalogue::setTapeLastFSeq; diff --git a/common/dataStructures/RetrieveFileQueueCriteria.cpp b/common/dataStructures/RetrieveFileQueueCriteria.cpp index e5f36ab167..961c1e3af6 100644 --- a/common/dataStructures/RetrieveFileQueueCriteria.cpp +++ b/common/dataStructures/RetrieveFileQueueCriteria.cpp @@ -32,7 +32,7 @@ RetrieveFileQueueCriteria::RetrieveFileQueueCriteria() { // constructor //------------------------------------------------------------------------------ RetrieveFileQueueCriteria::RetrieveFileQueueCriteria( - const std::list<TapeFile> &tapeFiles, + const std::map<uint64_t, TapeFile> &tapeFiles, const MountPolicy &mountPolicy): tapeFiles(tapeFiles), mountPolicy(mountPolicy) { diff --git a/common/dataStructures/RetrieveFileQueueCriteria.hpp b/common/dataStructures/RetrieveFileQueueCriteria.hpp index 61f349d744..06bc69328f 100644 --- a/common/dataStructures/RetrieveFileQueueCriteria.hpp +++ b/common/dataStructures/RetrieveFileQueueCriteria.hpp @@ -21,7 +21,8 @@ #include "common/dataStructures/MountPolicy.hpp" #include "common/dataStructures/TapeFile.hpp" -#include <list> +#include <map> +#include <stdint.h> namespace cta { namespace common { @@ -40,17 +41,19 @@ struct RetrieveFileQueueCriteria { /** * Constructor. * - * @param tapeFiles The location(s) of the file on tape. + * @param tapeFiles The location(s) of the file on tape as a map from tape + * copy number to location. * @param mountPolicy The mount policy. */ RetrieveFileQueueCriteria( - const std::list<TapeFile> &tapeFiles, + const std::map<uint64_t, TapeFile> &tapeFiles, const MountPolicy &mountPolicy); /** - * The location(s) of the file on tape. + * The location(s) of the file on tape as a map from tape copy number to + * location. */ - std::list<TapeFile> tapeFiles; + std::map<uint64_t, TapeFile> tapeFiles; /** * The mount policy. diff --git a/scheduler/OStoreDB/OStoreDB.cpp b/scheduler/OStoreDB/OStoreDB.cpp index a0cdfda44c..8ac08b0ae4 100644 --- a/scheduler/OStoreDB/OStoreDB.cpp +++ b/scheduler/OStoreDB/OStoreDB.cpp @@ -658,7 +658,8 @@ std::list<ArchiveToTapeCopyRequest> throw NoSuchArchiveQueue("In OStoreDB::getArchiveRequests: tape pool not found"); } -void OStoreDB::queue(const cta::RetrieveToFileRequest& rqst) { +void OStoreDB::queue(const cta::common::dataStructures::RetrieveRequest& rqst, + const cta::common::dataStructures::RetrieveFileQueueCriteria& criteria) { throw cta::exception::Exception(std::string("Not implemented: ") + __PRETTY_FUNCTION__); // assertAgentSet(); // // Check at least one potential tape copy is provided. diff --git a/scheduler/OStoreDB/OStoreDB.hpp b/scheduler/OStoreDB/OStoreDB.hpp index 6edbc2c502..6925e7b0b5 100644 --- a/scheduler/OStoreDB/OStoreDB.hpp +++ b/scheduler/OStoreDB/OStoreDB.hpp @@ -200,7 +200,8 @@ public: /* === Retrieve requests handling ======================================== */ CTA_GENERATE_EXCEPTION_CLASS(RetrieveRequestHasNoCopies); CTA_GENERATE_EXCEPTION_CLASS(TapeCopyNumberOutOfRange); - void queue(const RetrieveToFileRequest& rqst_) override; + void queue(const cta::common::dataStructures::RetrieveRequest& rqst, + const cta::common::dataStructures::RetrieveFileQueueCriteria &criteria) override; std::list<RetrieveRequestDump> getRetrieveRequests(const std::string& vid) const override; diff --git a/scheduler/OStoreDB/OStoreDBFactory.hpp b/scheduler/OStoreDB/OStoreDBFactory.hpp index 0c4d7b87b6..e7d1ca1bf9 100644 --- a/scheduler/OStoreDB/OStoreDBFactory.hpp +++ b/scheduler/OStoreDB/OStoreDBFactory.hpp @@ -90,8 +90,9 @@ public: return m_OStoreDB.getMountInfo(); } - virtual void queue(const RetrieveToFileRequest& rqst) override { - m_OStoreDB.queue(rqst); + virtual void queue(const common::dataStructures::RetrieveRequest& rqst, + const common::dataStructures::RetrieveFileQueueCriteria &criteria) override { + m_OStoreDB.queue(rqst, criteria); } virtual std::list<cta::common::DriveState> getDriveStates() const override { diff --git a/scheduler/Scheduler.cpp b/scheduler/Scheduler.cpp index c0de248300..0a2d07c847 100644 --- a/scheduler/Scheduler.cpp +++ b/scheduler/Scheduler.cpp @@ -29,7 +29,7 @@ #include <algorithm> //------------------------------------------------------------------------------ -// deprecated constructor +// constructor //------------------------------------------------------------------------------ cta::Scheduler::Scheduler( catalogue::Catalogue &catalogue, @@ -61,8 +61,12 @@ uint64_t cta::Scheduler::queueArchive(const cta::common::dataStructures::Securit //------------------------------------------------------------------------------ // queueRetrieve //------------------------------------------------------------------------------ -void cta::Scheduler::queueRetrieve(const cta::common::dataStructures::SecurityIdentity &cliIdentity, const cta::common::dataStructures::RetrieveRequest &request) { - throw cta::exception::Exception(std::string("Not implemented: ") + __PRETTY_FUNCTION__); +void cta::Scheduler::queueRetrieve( + const cta::common::dataStructures::SecurityIdentity &cliIdentity, + const cta::common::dataStructures::RetrieveRequest &request) { + const common::dataStructures::RetrieveFileQueueCriteria queueCriteria = + m_catalogue.prepareToRetrieveFile(request.archiveFileID, cliIdentity.user); + m_db.queue(request, queueCriteria); } //------------------------------------------------------------------------------ diff --git a/scheduler/SchedulerDatabase.hpp b/scheduler/SchedulerDatabase.hpp index 56f6d19824..b9ec4e9a70 100644 --- a/scheduler/SchedulerDatabase.hpp +++ b/scheduler/SchedulerDatabase.hpp @@ -18,27 +18,30 @@ #pragma once -#include <list> -#include <map> -#include <stdint.h> -#include <string> -#include <memory> -#include <vector> -#include <stdexcept> #include "common/archiveNS/ArchiveFile.hpp" #include "common/archiveNS/TapeFileLocation.hpp" #include "common/admin/AdminUser.hpp" #include "common/admin/AdminHost.hpp" #include "common/archiveRoutes/ArchiveRoute.hpp" -#include "common/remoteFS/RemotePathAndStatus.hpp" -#include "common/MountControl.hpp" +#include "common/dataStructures/RetrieveFileQueueCriteria.hpp" #include "common/DriveState.hpp" -#include "nameserver/NameServerTapeFile.hpp" -#include "scheduler/MountType.hpp" -#include "scheduler/ArchiveToTapeCopyRequest.hpp" +#include "common/MountControl.hpp" #include "common/dataStructures/ArchiveRequest.hpp" #include "common/dataStructures/ArchiveFileQueueCriteria.hpp" #include "common/dataStructures/MountPolicy.hpp" +#include "common/dataStructures/RetrieveRequest.hpp" +#include "common/remoteFS/RemotePathAndStatus.hpp" +#include "nameserver/NameServerTapeFile.hpp" +#include "scheduler/ArchiveToTapeCopyRequest.hpp" +#include "scheduler/MountType.hpp" + +#include <list> +#include <map> +#include <stdint.h> +#include <string> +#include <memory> +#include <vector> +#include <stdexcept> namespace cta { // Forward declarations for opaque references. @@ -104,6 +107,8 @@ public: * Queues the specified request. * * @param rqst The request. + * @param criteria The criteria retrieved from the CTA catalogue to be used to + * decide how to quue the request. */ virtual std::unique_ptr<ArchiveRequestCreation> queue(const cta::common::dataStructures::ArchiveRequest &request, const cta::common::dataStructures::ArchiveFileQueueCriteria &criteria) = 0; @@ -206,8 +211,11 @@ public: * Queues the specified request. * * @param rqst The request. + * @param criteria The criteria retrieved from the CTA catalogue to be used to + * decide how to quue the request. */ - virtual void queue(const RetrieveToFileRequest &rqst_) = 0; + virtual void queue(const cta::common::dataStructures::RetrieveRequest &rqst, + const cta::common::dataStructures::RetrieveFileQueueCriteria &criteria) = 0; /** * Returns all of the existing retrieve jobs grouped by tape and then -- GitLab