diff --git a/catalogue/CatalogueTest.cpp b/catalogue/CatalogueTest.cpp index 2d5134de5616daee80f865d35ba4a618a7a21ce9..5e3a6eb8690208fef0420dbdca148bdfa4c3901f 100644 --- a/catalogue/CatalogueTest.cpp +++ b/catalogue/CatalogueTest.cpp @@ -8677,68 +8677,15 @@ TEST_P(cta_catalogue_CatalogueTest, getArchiveFiles_non_existance_archiveFileId) ASSERT_THROW(m_catalogue->getArchiveFilesItor(searchCriteria), exception::UserError); } -TEST_P(cta_catalogue_CatalogueTest, getArchiveFiles_disk_file_group_without_instance) { - using namespace cta; - - ASSERT_FALSE(m_catalogue->getArchiveFilesItor().hasMore()); - - catalogue::TapeFileSearchCriteria searchCriteria; - searchCriteria.diskFileGid = DISK_FILE_GID; - - ASSERT_THROW(m_catalogue->getArchiveFilesItor(searchCriteria), exception::UserError); -} - -TEST_P(cta_catalogue_CatalogueTest, getArchiveFiles_non_existent_disk_file_group) { - using namespace cta; - ASSERT_FALSE(m_catalogue->getArchiveFilesItor().hasMore()); - - catalogue::TapeFileSearchCriteria searchCriteria; - searchCriteria.diskInstance = "non_existent_disk_instance"; - searchCriteria.diskFileGid = NON_EXISTENT_DISK_FILE_GID; - - ASSERT_THROW(m_catalogue->getArchiveFilesItor(searchCriteria), exception::UserError); -} - TEST_P(cta_catalogue_CatalogueTest, getArchiveFiles_disk_file_id_without_instance) { using namespace cta; ASSERT_FALSE(m_catalogue->getArchiveFilesItor().hasMore()); catalogue::TapeFileSearchCriteria searchCriteria; - searchCriteria.diskFileId = "disk_file_id"; - - ASSERT_THROW(m_catalogue->getArchiveFilesItor(searchCriteria), exception::UserError); -} - -TEST_P(cta_catalogue_CatalogueTest, getArchiveFiles_non_existent_disk_file_id) { - using namespace cta; - ASSERT_FALSE(m_catalogue->getArchiveFilesItor().hasMore()); - - catalogue::TapeFileSearchCriteria searchCriteria; - searchCriteria.diskInstance = "non_existent_disk_instance"; - searchCriteria.diskFileId = "non_existent_disk_file_id"; - - ASSERT_THROW(m_catalogue->getArchiveFilesItor(searchCriteria), exception::UserError); -} - -TEST_P(cta_catalogue_CatalogueTest, getArchiveFiles_disk_file_user_without_instance) { - using namespace cta; - - ASSERT_FALSE(m_catalogue->getArchiveFilesItor().hasMore()); - - catalogue::TapeFileSearchCriteria searchCriteria; - searchCriteria.diskFileOwnerUid = DISK_FILE_OWNER_UID; - - ASSERT_THROW(m_catalogue->getArchiveFilesItor(searchCriteria), exception::UserError); -} - -TEST_P(cta_catalogue_CatalogueTest, getArchiveFiles_non_existent_disk_file_user) { - using namespace cta; - ASSERT_FALSE(m_catalogue->getArchiveFilesItor().hasMore()); - - catalogue::TapeFileSearchCriteria searchCriteria; - searchCriteria.diskInstance = "non_existent_disk_instance"; - searchCriteria.diskFileOwnerUid = NON_EXISTENT_DISK_FILE_OWNER_UID; + std::vector<std::string> diskFileIds; + diskFileIds.push_back("disk_file_id"); + searchCriteria.diskFileIds = diskFileIds; ASSERT_THROW(m_catalogue->getArchiveFilesItor(searchCriteria), exception::UserError); } @@ -8769,34 +8716,6 @@ TEST_P(cta_catalogue_CatalogueTest, getArchiveFiles_existent_storage_class_witho } ASSERT_FALSE(m_catalogue->getArchiveFilesItor().hasMore()); - - catalogue::TapeFileSearchCriteria searchCriteria; - searchCriteria.storageClass = m_storageClassSingleCopy.name; - - ASSERT_THROW(m_catalogue->getArchiveFilesItor(searchCriteria), exception::UserError); -} - -TEST_P(cta_catalogue_CatalogueTest, getArchiveFiles_non_existent_storage_class) { - using namespace cta; - - ASSERT_FALSE(m_catalogue->getArchiveFilesItor().hasMore()); - - catalogue::TapeFileSearchCriteria searchCriteria; - searchCriteria.diskInstance = "non_existent_disk_instance"; - searchCriteria.storageClass = "non_existent_storage_class"; - - ASSERT_THROW(m_catalogue->getArchiveFilesItor(searchCriteria), exception::UserError); -} - -TEST_P(cta_catalogue_CatalogueTest, getArchiveFiles_non_existent_tape_pool) { - using namespace cta; - - ASSERT_FALSE(m_catalogue->getArchiveFilesItor().hasMore()); - - catalogue::TapeFileSearchCriteria searchCriteria; - searchCriteria.tapePool = "non_existent_tape_pool"; - - ASSERT_THROW(m_catalogue->getArchiveFilesItor(searchCriteria), exception::UserError); } TEST_P(cta_catalogue_CatalogueTest, getArchiveFiles_non_existent_vid) { @@ -9093,13 +9012,10 @@ TEST_P(cta_catalogue_CatalogueTest, filesWrittenToTape_many_archive_files) { catalogue::TapeFileSearchCriteria searchCriteria; searchCriteria.archiveFileId = 1; searchCriteria.diskInstance = diskInstance; - searchCriteria.diskFileId = std::to_string(12345678); - searchCriteria.diskFileOwnerUid = PUBLIC_DISK_USER; - searchCriteria.diskFileGid = PUBLIC_DISK_GROUP; - searchCriteria.storageClass = m_storageClassDualCopy.name; + std::vector<std::string> diskFileIds; + diskFileIds.push_back("12345678"); + searchCriteria.diskFileIds = diskFileIds; searchCriteria.vid = tape1.vid; - searchCriteria.tapeFileCopyNb = 1; - searchCriteria.tapePool = tapePoolName1; auto archiveFileItor = m_catalogue->getArchiveFilesItor(searchCriteria); std::map<uint64_t, common::dataStructures::ArchiveFile> m = archiveFileItorToMap(archiveFileItor); @@ -9110,10 +9026,7 @@ TEST_P(cta_catalogue_CatalogueTest, filesWrittenToTape_many_archive_files) { const common::dataStructures::ArchiveFile archiveFile = idAndFile->second; ASSERT_EQ(searchCriteria.archiveFileId, archiveFile.archiveFileID); ASSERT_EQ(searchCriteria.diskInstance, archiveFile.diskInstance); - ASSERT_EQ(searchCriteria.diskFileId, archiveFile.diskFileId); - ASSERT_EQ(searchCriteria.diskFileOwnerUid, static_cast<uint64_t>(archiveFile.diskFileInfo.owner_uid)); - ASSERT_EQ(searchCriteria.diskFileGid, static_cast<uint64_t>(archiveFile.diskFileInfo.gid)); - ASSERT_EQ(searchCriteria.storageClass, archiveFile.storageClass); + ASSERT_EQ(searchCriteria.diskFileIds->front(), archiveFile.diskFileId); ASSERT_EQ(1, archiveFile.tapeFiles.size()); ASSERT_EQ(searchCriteria.vid, archiveFile.tapeFiles.begin()->vid); } @@ -9458,7 +9371,9 @@ TEST_P(cta_catalogue_CatalogueTest, filesWrittenToTape_many_archive_files) { { catalogue::TapeFileSearchCriteria searchCriteria; searchCriteria.diskInstance = diskInstance; - searchCriteria.diskFileId = "12345687"; + std::vector<std::string> diskFileIds; + diskFileIds.push_back("12345687"); + searchCriteria.diskFileIds = diskFileIds; auto archiveFileItor = m_catalogue->getArchiveFilesItor(searchCriteria); const auto m = archiveFileItorToMap(archiveFileItor); ASSERT_EQ(1, m.size()); @@ -9469,45 +9384,6 @@ TEST_P(cta_catalogue_CatalogueTest, filesWrittenToTape_many_archive_files) { ASSERT_EQ(m_storageClassDualCopy.nbCopies, summary.totalFiles); } - { - catalogue::TapeFileSearchCriteria searchCriteria; - searchCriteria.diskInstance = diskInstance; - searchCriteria.diskFileOwnerUid = PUBLIC_DISK_USER; - auto archiveFileItor = m_catalogue->getArchiveFilesItor(searchCriteria); - const auto m = archiveFileItorToMap(archiveFileItor); - ASSERT_EQ(nbArchiveFiles, m.size()); - - const common::dataStructures::ArchiveFileSummary summary = m_catalogue->getTapeFileSummary(searchCriteria); - ASSERT_EQ(nbArchiveFiles * m_storageClassDualCopy.nbCopies * archiveFileSize, summary.totalBytes); - ASSERT_EQ(nbArchiveFiles * m_storageClassDualCopy.nbCopies, summary.totalFiles); - } - - { - catalogue::TapeFileSearchCriteria searchCriteria; - searchCriteria.diskInstance = diskInstance; - searchCriteria.diskFileGid = PUBLIC_DISK_GROUP; - auto archiveFileItor = m_catalogue->getArchiveFilesItor(searchCriteria); - const auto m = archiveFileItorToMap(archiveFileItor); - ASSERT_EQ(nbArchiveFiles, m.size()); - - const common::dataStructures::ArchiveFileSummary summary = m_catalogue->getTapeFileSummary(searchCriteria); - ASSERT_EQ(nbArchiveFiles * m_storageClassDualCopy.nbCopies * archiveFileSize, summary.totalBytes); - ASSERT_EQ(nbArchiveFiles * m_storageClassDualCopy.nbCopies, summary.totalFiles); - } - - { - catalogue::TapeFileSearchCriteria searchCriteria; - searchCriteria.diskInstance = diskInstance; - searchCriteria.storageClass = m_storageClassDualCopy.name; - auto archiveFileItor = m_catalogue->getArchiveFilesItor(searchCriteria); - const auto m = archiveFileItorToMap(archiveFileItor); - ASSERT_EQ(nbArchiveFiles, m.size()); - - const common::dataStructures::ArchiveFileSummary summary = m_catalogue->getTapeFileSummary(searchCriteria); - ASSERT_EQ(nbArchiveFiles * m_storageClassDualCopy.nbCopies * archiveFileSize, summary.totalBytes); - ASSERT_EQ(nbArchiveFiles * m_storageClassDualCopy.nbCopies, summary.totalFiles); - } - { catalogue::TapeFileSearchCriteria searchCriteria; auto archiveFileItor = m_catalogue->getArchiveFilesItor(searchCriteria); @@ -9531,42 +9407,6 @@ TEST_P(cta_catalogue_CatalogueTest, filesWrittenToTape_many_archive_files) { ASSERT_EQ(nbArchiveFiles, summary.totalFiles); } - { - catalogue::TapeFileSearchCriteria searchCriteria; - searchCriteria.tapeFileCopyNb = 1; - auto archiveFileItor = m_catalogue->getArchiveFilesItor(searchCriteria); - const auto m = archiveFileItorToMap(archiveFileItor); - ASSERT_EQ(nbArchiveFiles, m.size()); - - const common::dataStructures::ArchiveFileSummary summary = m_catalogue->getTapeFileSummary(searchCriteria); - ASSERT_EQ(nbArchiveFiles * archiveFileSize, summary.totalBytes); - ASSERT_EQ(nbArchiveFiles, summary.totalFiles); - } - - { - catalogue::TapeFileSearchCriteria searchCriteria; - searchCriteria.tapePool = tapePoolName1; - auto archiveFileItor = m_catalogue->getArchiveFilesItor(searchCriteria); - const auto m = archiveFileItorToMap(archiveFileItor); - ASSERT_EQ(nbArchiveFiles, m.size()); - - const common::dataStructures::ArchiveFileSummary summary = m_catalogue->getTapeFileSummary(searchCriteria); - ASSERT_EQ(nbArchiveFiles * archiveFileSize, summary.totalBytes); - ASSERT_EQ(nbArchiveFiles, summary.totalFiles); - } - - { - catalogue::TapeFileSearchCriteria searchCriteria; - searchCriteria.tapePool = tapePoolName2; - auto archiveFileItor = m_catalogue->getArchiveFilesItor(searchCriteria); - const auto m = archiveFileItorToMap(archiveFileItor); - ASSERT_EQ(nbArchiveFiles, m.size()); - - const common::dataStructures::ArchiveFileSummary summary = m_catalogue->getTapeFileSummary(searchCriteria); - ASSERT_EQ(nbArchiveFiles * archiveFileSize, summary.totalBytes); - ASSERT_EQ(nbArchiveFiles, summary.totalFiles); - } - { catalogue::TapeFileSearchCriteria searchCriteria; searchCriteria.archiveFileId = nbArchiveFiles + 1234; @@ -9975,13 +9815,10 @@ TEST_P(cta_catalogue_CatalogueTest, DISABLED_concurrent_filesWrittenToTape_many_ catalogue::TapeFileSearchCriteria searchCriteria; searchCriteria.archiveFileId = 1; searchCriteria.diskInstance = diskInstance; - searchCriteria.diskFileId = std::to_string(12345678); - searchCriteria.diskFileOwnerUid = PUBLIC_DISK_USER; - searchCriteria.diskFileGid = PUBLIC_DISK_GROUP; - searchCriteria.storageClass = storageClass.name; + std::vector<std::string> diskFileIds; + diskFileIds.push_back("12345678"); + searchCriteria.diskFileIds = diskFileIds; searchCriteria.vid = tape1.vid; - searchCriteria.tapeFileCopyNb = 1; - searchCriteria.tapePool = tapePoolName1; auto archiveFileItor = m_catalogue->getArchiveFilesItor(searchCriteria); std::map<uint64_t, common::dataStructures::ArchiveFile> m = archiveFileItorToMap(archiveFileItor); @@ -9992,10 +9829,7 @@ TEST_P(cta_catalogue_CatalogueTest, DISABLED_concurrent_filesWrittenToTape_many_ const common::dataStructures::ArchiveFile archiveFile = idAndFile->second; ASSERT_EQ(searchCriteria.archiveFileId, archiveFile.archiveFileID); ASSERT_EQ(searchCriteria.diskInstance, archiveFile.diskInstance); - ASSERT_EQ(searchCriteria.diskFileId, archiveFile.diskFileId); - ASSERT_EQ(searchCriteria.diskFileOwnerUid, static_cast<uint64_t>(archiveFile.diskFileInfo.owner_uid)); - ASSERT_EQ(searchCriteria.diskFileGid, static_cast<uint64_t>(archiveFile.diskFileInfo.gid)); - ASSERT_EQ(searchCriteria.storageClass, archiveFile.storageClass); + ASSERT_EQ(searchCriteria.diskFileIds->front(), archiveFile.diskFileId); ASSERT_EQ(1, archiveFile.tapeFiles.size()); ASSERT_EQ(searchCriteria.vid, archiveFile.tapeFiles.begin()->vid); } @@ -10369,7 +10203,9 @@ TEST_P(cta_catalogue_CatalogueTest, DISABLED_concurrent_filesWrittenToTape_many_ { catalogue::TapeFileSearchCriteria searchCriteria; searchCriteria.diskInstance = diskInstance; - searchCriteria.diskFileId = "12345687"; + std::vector<std::string> diskFileIds; + diskFileIds.push_back("12345687"); + searchCriteria.diskFileIds = diskFileIds; auto archiveFileItor = m_catalogue->getArchiveFilesItor(searchCriteria); const auto m = archiveFileItorToMap(archiveFileItor); ASSERT_EQ(1, m.size()); @@ -10393,45 +10229,6 @@ TEST_P(cta_catalogue_CatalogueTest, DISABLED_concurrent_filesWrittenToTape_many_ ASSERT_EQ(storageClass.nbCopies, summary.totalFiles); } - { - catalogue::TapeFileSearchCriteria searchCriteria; - searchCriteria.diskInstance = diskInstance; - searchCriteria.diskFileOwnerUid = PUBLIC_DISK_USER; - auto archiveFileItor = m_catalogue->getArchiveFilesItor(searchCriteria); - const auto m = archiveFileItorToMap(archiveFileItor); - ASSERT_EQ(nbArchiveFiles, m.size()); - - const common::dataStructures::ArchiveFileSummary summary = m_catalogue->getTapeFileSummary(searchCriteria); - ASSERT_EQ(nbArchiveFiles * storageClass.nbCopies * archiveFileSize, summary.totalBytes); - ASSERT_EQ(nbArchiveFiles * storageClass.nbCopies, summary.totalFiles); - } - - { - catalogue::TapeFileSearchCriteria searchCriteria; - searchCriteria.diskInstance = diskInstance; - searchCriteria.diskFileGid = PUBLIC_DISK_GROUP; - auto archiveFileItor = m_catalogue->getArchiveFilesItor(searchCriteria); - const auto m = archiveFileItorToMap(archiveFileItor); - ASSERT_EQ(nbArchiveFiles, m.size()); - - const common::dataStructures::ArchiveFileSummary summary = m_catalogue->getTapeFileSummary(searchCriteria); - ASSERT_EQ(nbArchiveFiles * storageClass.nbCopies * archiveFileSize, summary.totalBytes); - ASSERT_EQ(nbArchiveFiles * storageClass.nbCopies, summary.totalFiles); - } - - { - catalogue::TapeFileSearchCriteria searchCriteria; - searchCriteria.diskInstance = diskInstance; - searchCriteria.storageClass = storageClass.name; - auto archiveFileItor = m_catalogue->getArchiveFilesItor(searchCriteria); - const auto m = archiveFileItorToMap(archiveFileItor); - ASSERT_EQ(nbArchiveFiles, m.size()); - - const common::dataStructures::ArchiveFileSummary summary = m_catalogue->getTapeFileSummary(searchCriteria); - ASSERT_EQ(nbArchiveFiles * storageClass.nbCopies * archiveFileSize, summary.totalBytes); - ASSERT_EQ(nbArchiveFiles * storageClass.nbCopies, summary.totalFiles); - } - { catalogue::TapeFileSearchCriteria searchCriteria; auto archiveFileItor = m_catalogue->getArchiveFilesItor(searchCriteria); @@ -10455,42 +10252,6 @@ TEST_P(cta_catalogue_CatalogueTest, DISABLED_concurrent_filesWrittenToTape_many_ ASSERT_EQ(nbArchiveFiles, summary.totalFiles); } - { - catalogue::TapeFileSearchCriteria searchCriteria; - searchCriteria.tapeFileCopyNb = 1; - auto archiveFileItor = m_catalogue->getArchiveFilesItor(searchCriteria); - const auto m = archiveFileItorToMap(archiveFileItor); - ASSERT_EQ(nbArchiveFiles, m.size()); - - const common::dataStructures::ArchiveFileSummary summary = m_catalogue->getTapeFileSummary(searchCriteria); - ASSERT_EQ(nbArchiveFiles * archiveFileSize, summary.totalBytes); - ASSERT_EQ(nbArchiveFiles, summary.totalFiles); - } - - { - catalogue::TapeFileSearchCriteria searchCriteria; - searchCriteria.tapePool = tapePoolName1; - auto archiveFileItor = m_catalogue->getArchiveFilesItor(searchCriteria); - const auto m = archiveFileItorToMap(archiveFileItor); - ASSERT_EQ(nbArchiveFiles, m.size()); - - const common::dataStructures::ArchiveFileSummary summary = m_catalogue->getTapeFileSummary(searchCriteria); - ASSERT_EQ(nbArchiveFiles * archiveFileSize, summary.totalBytes); - ASSERT_EQ(nbArchiveFiles, summary.totalFiles); - } - - { - catalogue::TapeFileSearchCriteria searchCriteria; - searchCriteria.tapePool = tapePoolName2; - auto archiveFileItor = m_catalogue->getArchiveFilesItor(searchCriteria); - const auto m = archiveFileItorToMap(archiveFileItor); - ASSERT_EQ(nbArchiveFiles, m.size()); - - const common::dataStructures::ArchiveFileSummary summary = m_catalogue->getTapeFileSummary(searchCriteria); - ASSERT_EQ(nbArchiveFiles * archiveFileSize, summary.totalBytes); - ASSERT_EQ(nbArchiveFiles, summary.totalFiles); - } - { catalogue::TapeFileSearchCriteria searchCriteria; searchCriteria.archiveFileId = nbArchiveFiles + 1234; diff --git a/catalogue/MysqlCatalogue.cpp b/catalogue/MysqlCatalogue.cpp index 31c12693db72445d274234c42c4d81cdddf504a9..03849a7188d7b9d951e39ff0eb5e3cba6f586d0e 100644 --- a/catalogue/MysqlCatalogue.cpp +++ b/catalogue/MysqlCatalogue.cpp @@ -56,6 +56,38 @@ MysqlCatalogue::MysqlCatalogue( MysqlCatalogue::~MysqlCatalogue() { } +//------------------------------------------------------------------------------ +// createAndPopulateTempTableFxid +//------------------------------------------------------------------------------ +std::string MysqlCatalogue::createAndPopulateTempTableFxid(rdbms::Conn &conn, const TapeFileSearchCriteria &tapeFileSearchCriteria) const { + const std::string tempTableName = "TEMP_DISK_FXIDS"; + + if(tapeFileSearchCriteria.diskFileIds) { + try { + std::string sql = "CREATE TEMPORARY TABLE " + tempTableName + "(DISK_FILE_ID VARCHAR(100))"; + try { + conn.executeNonQuery(sql); + } catch(exception::Exception &ex) { + // MySQL does not drop temporary tables until the end of the session; trying to create another + // temporary table in the same unit test will fail. If this happens, truncate the table and carry on. + sql = "TRUNCATE TABLE " + tempTableName; + conn.executeNonQuery(sql); + } + + sql = "INSERT INTO " + tempTableName + " VALUES(:DISK_FILE_ID)"; + auto stmt = conn.createStmt(sql); + for(auto &diskFileId : tapeFileSearchCriteria.diskFileIds.value()) { + stmt.bindString(":DISK_FILE_ID", diskFileId); + stmt.executeNonQuery(); + } + } catch(exception::Exception &ex) { + ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str()); + throw; + } + } + return tempTableName; +} + //------------------------------------------------------------------------------ // getNextArchiveFileId //------------------------------------------------------------------------------ diff --git a/catalogue/MysqlCatalogue.hpp b/catalogue/MysqlCatalogue.hpp index 5ed59844584a3867256ebee3faa999f3b0c74855..702d2399aa3a4d66560fed4719c46bb4282e0f46 100644 --- a/catalogue/MysqlCatalogue.hpp +++ b/catalogue/MysqlCatalogue.hpp @@ -57,6 +57,15 @@ public: protected: + /** + * Creates a temporary table from the list of disk file IDs provided in the search criteria. + * + * @param conn The database connection. + * @param tapeFileSearchCriteria Search criteria containing a list of disk file IDs (fxid). + * @return Name of the temporary table + */ + std::string createAndPopulateTempTableFxid(rdbms::Conn &conn, const TapeFileSearchCriteria &tapeFileSearchCriteria) const override; + /** * Returns a unique archive ID that can be used by a new archive file within * the catalogue. diff --git a/catalogue/OracleCatalogue.cpp b/catalogue/OracleCatalogue.cpp index c0983a97a5685ac3e2e520b41fead491e4d92ebc..c1d42a77609fd3c16d17051c4d8cacf872a495ac 100644 --- a/catalogue/OracleCatalogue.cpp +++ b/catalogue/OracleCatalogue.cpp @@ -151,6 +151,34 @@ OracleCatalogue::OracleCatalogue( OracleCatalogue::~OracleCatalogue() { } +//------------------------------------------------------------------------------ +// createAndPopulateTempTableFxid +//------------------------------------------------------------------------------ +std::string OracleCatalogue::createAndPopulateTempTableFxid(rdbms::Conn &conn, const TapeFileSearchCriteria &tapeFileSearchCriteria) const { + const std::string tempTableName = "ORA$PTT_DISK_FXIDS"; + + try { + if(tapeFileSearchCriteria.diskFileIds) { + conn.setAutocommitMode(rdbms::AutocommitMode::AUTOCOMMIT_OFF); + std::string sql = "CREATE PRIVATE TEMPORARY TABLE " + tempTableName + + "(DISK_FILE_ID VARCHAR2(100))"; + conn.executeNonQuery(sql); + + sql = "INSERT INTO " + tempTableName + " VALUES(:DISK_FILE_ID)"; + auto stmt = conn.createStmt(sql); + for(auto &diskFileId : tapeFileSearchCriteria.diskFileIds.value()) { + stmt.bindString(":DISK_FILE_ID", diskFileId); + stmt.executeNonQuery(); + } + } + + return tempTableName; + } catch(exception::Exception &ex) { + ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str()); + throw; + } +} + //------------------------------------------------------------------------------ // getNextArchiveFileId //------------------------------------------------------------------------------ diff --git a/catalogue/OracleCatalogue.hpp b/catalogue/OracleCatalogue.hpp index d7e2bde4ec429c5f3724703805ab79af640befa3..ef013b7a53ab80ed4d75e98c4490f9870f93866c 100644 --- a/catalogue/OracleCatalogue.hpp +++ b/catalogue/OracleCatalogue.hpp @@ -57,6 +57,15 @@ public: */ ~OracleCatalogue() override; + /** + * Creates a temporary table from the list of disk file IDs provided in the search criteria. + * + * @param conn The database connection. + * @param tapeFileSearchCriteria Search criteria containing a list of disk file IDs (fxid). + * @return Name of the temporary table + */ + std::string createAndPopulateTempTableFxid(rdbms::Conn &conn, const TapeFileSearchCriteria &tapeFileSearchCriteria) const override; + /** * Returns a unique archive ID that can be used by a new archive file within * the catalogue. diff --git a/catalogue/PostgresCatalogue.cpp b/catalogue/PostgresCatalogue.cpp index 210fb5269b8c32d1e6bd989cce579412ab300b75..8469ab9cfe24fe9ef0aa18927c32c334081ef3cd 100644 --- a/catalogue/PostgresCatalogue.cpp +++ b/catalogue/PostgresCatalogue.cpp @@ -149,6 +149,38 @@ PostgresCatalogue::PostgresCatalogue( PostgresCatalogue::~PostgresCatalogue() { } +//------------------------------------------------------------------------------ +// createAndPopulateTempTableFxid +//------------------------------------------------------------------------------ +std::string PostgresCatalogue::createAndPopulateTempTableFxid(rdbms::Conn &conn, const TapeFileSearchCriteria &tapeFileSearchCriteria) const { + const std::string tempTableName = "TEMP_DISK_FXIDS"; + + if(tapeFileSearchCriteria.diskFileIds) { + try { + std::string sql = "CREATE TEMPORARY TABLE " + tempTableName + "(DISK_FILE_ID VARCHAR(100))"; + try { + conn.executeNonQuery(sql); + } catch(exception::Exception &ex) { + // Postgres does not drop temporary tables until the end of the session; trying to create another + // temporary table in the same unit test will fail. If this happens, truncate the table and carry on. + sql = "TRUNCATE TABLE " + tempTableName; + conn.executeNonQuery(sql); + } + + sql = "INSERT INTO " + tempTableName + " VALUES(:DISK_FILE_ID)"; + auto stmt = conn.createStmt(sql); + for(auto &diskFileId : tapeFileSearchCriteria.diskFileIds.value()) { + stmt.bindString(":DISK_FILE_ID", diskFileId); + stmt.executeNonQuery(); + } + } catch(exception::Exception &ex) { + ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str()); + throw; + } + } + return tempTableName; +} + //------------------------------------------------------------------------------ // getNextArchiveFileId //------------------------------------------------------------------------------ diff --git a/catalogue/PostgresCatalogue.hpp b/catalogue/PostgresCatalogue.hpp index ab405c8865abd253ba8d56df4b477881e3be84b6..b6d28a049d18d81399d6919d385a9852cef6754c 100644 --- a/catalogue/PostgresCatalogue.hpp +++ b/catalogue/PostgresCatalogue.hpp @@ -84,6 +84,15 @@ public: */ void filesWrittenToTape(const std::set<TapeItemWrittenPointer> &events) override; + /** + * Creates a temporary table from the list of disk file IDs provided in the search criteria. + * + * @param conn The database connection. + * @param tapeFileSearchCriteria Search criteria containing a list of disk file IDs (fxid). + * @return Name of the temporary table + */ + std::string createAndPopulateTempTableFxid(rdbms::Conn &conn, const TapeFileSearchCriteria &tapeFileSearchCriteria) const override; + /** * Returns a unique archive ID that can be used by a new archive file within * the catalogue. diff --git a/catalogue/RdbmsCatalogue.cpp b/catalogue/RdbmsCatalogue.cpp index 103d037864bffd33a1f784e61fdd113f0dc698dd..d04b73fd1afbedd10c6a6679066835e4eaa7c9ea 100644 --- a/catalogue/RdbmsCatalogue.cpp +++ b/catalogue/RdbmsCatalogue.cpp @@ -6530,58 +6530,8 @@ void RdbmsCatalogue::checkTapeFileSearchCriteria(const TapeFileSearchCriteria &s } } - if(searchCriteria.diskFileGid && !searchCriteria.diskInstance) { - throw exception::UserError(std::string("Disk file group ") + std::to_string(searchCriteria.diskFileGid.value()) + - " is ambiguous without disk instance name"); - } - - if(searchCriteria.diskInstance && searchCriteria.diskFileGid) { - if(!diskFileGroupExists(conn, searchCriteria.diskInstance.value(), searchCriteria.diskFileGid.value())) { - throw exception::UserError(std::string("Disk file group ") + searchCriteria.diskInstance.value() + "::" + - std::to_string(searchCriteria.diskFileGid.value()) + " does not exist"); - } - } - - if(searchCriteria.diskFileId && !searchCriteria.diskInstance) { - throw exception::UserError(std::string("Disk file ID ") + searchCriteria.diskFileId.value() + " is ambiguous " - "without disk instance name"); - } - - if(searchCriteria.diskInstance && searchCriteria.diskFileId) { - if(!diskFileIdExists(conn, searchCriteria.diskInstance.value(), searchCriteria.diskFileId.value())) { - throw exception::UserError(std::string("Disk file ID ") + searchCriteria.diskInstance.value() + "::" + - searchCriteria.diskFileId.value() + " does not exist"); - } - } - - if(searchCriteria.diskFileOwnerUid && !searchCriteria.diskInstance) { - throw exception::UserError(std::string("Disk file user ") + std::to_string(searchCriteria.diskFileOwnerUid.value()) + - " is ambiguous without disk instance name"); - } - - if(searchCriteria.diskInstance && searchCriteria.diskFileOwnerUid) { - if(!diskFileUserExists(conn, searchCriteria.diskInstance.value(), searchCriteria.diskFileOwnerUid.value())) { - throw exception::UserError(std::string("Disk file user ") + searchCriteria.diskInstance.value() + "::" + - std::to_string(searchCriteria.diskFileOwnerUid.value()) + " does not exist"); - } - } - - if(searchCriteria.storageClass && !searchCriteria.diskInstance) { - throw exception::UserError(std::string("Storage class ") + searchCriteria.storageClass.value() + " is ambiguous " - "without disk instance name"); - } - - if(searchCriteria.diskInstance && searchCriteria.storageClass) { - if(!storageClassExists(conn, searchCriteria.storageClass.value())) { - throw exception::UserError(std::string("Storage class ") + "::" + - searchCriteria.storageClass.value() + " does not exist"); - } - } - - if(searchCriteria.tapePool) { - if(!tapePoolExists(conn, searchCriteria.tapePool.value())) { - throw exception::UserError(std::string("Tape pool ") + searchCriteria.tapePool.value() + " does not exist"); - } + if(searchCriteria.diskFileIds && !searchCriteria.diskInstance) { + throw exception::UserError(std::string("Disk file IDs are ambiguous without disk instance name")); } if(searchCriteria.vid) { @@ -6599,7 +6549,11 @@ ArchiveFileItor RdbmsCatalogue::getArchiveFilesItor(const TapeFileSearchCriteria checkTapeFileSearchCriteria(searchCriteria); try { - auto impl = new RdbmsCatalogueGetArchiveFilesItor(m_log, m_archiveFileListingConnPool, searchCriteria); + // Create a connection to populate the temporary table (specialised by database type) + auto conn = m_archiveFileListingConnPool.getConn(); + const auto tempDiskFxidsTableName = createAndPopulateTempTableFxid(conn, searchCriteria); + // Pass ownership of the connection to the Iterator object + auto impl = new RdbmsCatalogueGetArchiveFilesItor(m_log, std::move(conn), searchCriteria, tempDiskFxidsTableName); return ArchiveFileItor(impl); } catch(exception::UserError &) { throw; @@ -6739,10 +6693,17 @@ ArchiveFileItor RdbmsCatalogue::getArchiveFilesForRepackItor(const std::string & //------------------------------------------------------------------------------ // getTapeFileSummary +// +// NOTE: As "archivefile ls" has been deprecated, there is no longer a way for +// operators to request a tape file summary. (Use "tape ls" instead). +// This method is used exclusively by the unit tests. //------------------------------------------------------------------------------ common::dataStructures::ArchiveFileSummary RdbmsCatalogue::getTapeFileSummary( - const TapeFileSearchCriteria &searchCriteria) const { + const TapeFileSearchCriteria &searchCriteria) const +{ try { + auto conn = m_connPool.getConn(); + std::string sql = "SELECT " "COALESCE(SUM(ARCHIVE_FILE.SIZE_IN_BYTES), 0) AS TOTAL_BYTES," @@ -6758,16 +6719,15 @@ common::dataStructures::ArchiveFileSummary RdbmsCatalogue::getTapeFileSummary( "INNER JOIN TAPE_POOL ON " "TAPE.TAPE_POOL_ID = TAPE_POOL.TAPE_POOL_ID"; - if( + const bool hideSuperseded = searchCriteria.showSuperseded ? !*searchCriteria.showSuperseded : false; + const bool thereIsAtLeastOneSearchCriteria = searchCriteria.archiveFileId || searchCriteria.diskInstance || - searchCriteria.diskFileId || - searchCriteria.diskFileOwnerUid || - searchCriteria.diskFileGid || - searchCriteria.storageClass || searchCriteria.vid || - searchCriteria.tapeFileCopyNb || - searchCriteria.tapePool) { + searchCriteria.diskFileIds || + hideSuperseded; + + if(thereIsAtLeastOneSearchCriteria) { sql += " WHERE "; } @@ -6782,42 +6742,24 @@ common::dataStructures::ArchiveFileSummary RdbmsCatalogue::getTapeFileSummary( sql += "ARCHIVE_FILE.DISK_INSTANCE_NAME = :DISK_INSTANCE_NAME"; addedAWhereConstraint = true; } - if(searchCriteria.diskFileId) { - if(addedAWhereConstraint) sql += " AND "; - sql += "ARCHIVE_FILE.DISK_FILE_ID = :DISK_FILE_ID"; - addedAWhereConstraint = true; - } - if(searchCriteria.diskFileOwnerUid) { - if(addedAWhereConstraint) sql += " AND "; - sql += "ARCHIVE_FILE.DISK_FILE_UID = :DISK_FILE_UID"; - addedAWhereConstraint = true; - } - if(searchCriteria.diskFileGid) { - if(addedAWhereConstraint) sql += " AND "; - sql += "ARCHIVE_FILE.DISK_FILE_GID = :DISK_FILE_GID"; - addedAWhereConstraint = true; - } - if(searchCriteria.storageClass) { - if(addedAWhereConstraint) sql += " AND "; - sql += "STORAGE_CLASS.STORAGE_CLASS_NAME = :STORAGE_CLASS_NAME"; - addedAWhereConstraint = true; - } if(searchCriteria.vid) { if(addedAWhereConstraint) sql += " AND "; sql += "TAPE_FILE.VID = :VID"; addedAWhereConstraint = true; } - if(searchCriteria.tapeFileCopyNb) { + if(searchCriteria.diskFileIds) { + const auto tempDiskFxidsTableName = createAndPopulateTempTableFxid(conn, searchCriteria); + if(addedAWhereConstraint) sql += " AND "; - sql += "TAPE_FILE.COPY_NB = :TAPE_FILE_COPY_NB"; + sql += "ARCHIVE_FILE.DISK_FILE_ID IN (SELECT DISK_FILE_ID FROM " + tempDiskFxidsTableName + ")"; addedAWhereConstraint = true; } - if(searchCriteria.tapePool) { + if(hideSuperseded) { if(addedAWhereConstraint) sql += " AND "; - sql += "TAPE_POOL.TAPE_POOL_NAME = :TAPE_POOL_NAME"; + sql += "TAPE_FILE.SUPERSEDED_BY_VID IS NULL"; + addedAWhereConstraint = true; } - auto conn = m_connPool.getConn(); auto stmt = conn.createStmt(sql); if(searchCriteria.archiveFileId) { stmt.bindUint64(":ARCHIVE_FILE_ID", searchCriteria.archiveFileId.value()); @@ -6825,30 +6767,13 @@ common::dataStructures::ArchiveFileSummary RdbmsCatalogue::getTapeFileSummary( if(searchCriteria.diskInstance) { stmt.bindString(":DISK_INSTANCE_NAME", searchCriteria.diskInstance.value()); } - if(searchCriteria.diskFileId) { - stmt.bindString(":DISK_FILE_ID", searchCriteria.diskFileId.value()); - } - if(searchCriteria.diskFileOwnerUid) { - stmt.bindUint64(":DISK_FILE_UID", searchCriteria.diskFileOwnerUid.value()); - } - if(searchCriteria.diskFileGid) { - stmt.bindUint64(":DISK_FILE_GID", searchCriteria.diskFileGid.value()); - } - if(searchCriteria.storageClass) { - stmt.bindString(":STORAGE_CLASS_NAME", searchCriteria.storageClass.value()); - } if(searchCriteria.vid) { stmt.bindString(":VID", searchCriteria.vid.value()); } - if(searchCriteria.tapeFileCopyNb) { - stmt.bindUint64(":TAPE_FILE_COPY_NB", searchCriteria.tapeFileCopyNb.value()); - } - if(searchCriteria.tapePool) { - stmt.bindString(":TAPE_POOL_NAME", searchCriteria.tapePool.value()); - } auto rset = stmt.executeQuery(); + if(!rset.next()) { - throw exception::Exception("SELECT COUNT statement did not returned a row"); + throw exception::Exception("SELECT COUNT statement did not return a row"); } common::dataStructures::ArchiveFileSummary summary; diff --git a/catalogue/RdbmsCatalogue.hpp b/catalogue/RdbmsCatalogue.hpp index ced57edb69a5299a7ca2b063bf69b41731815cc9..77c09342f0d11dab489d2ce193ae7fca61d97e58 100644 --- a/catalogue/RdbmsCatalogue.hpp +++ b/catalogue/RdbmsCatalogue.hpp @@ -1519,6 +1519,15 @@ protected: const std::string &requesterName, const std::string &requesterGroupName) const; + /** + * Creates a temporary table from the list of disk file IDs provided in the search criteria. + * + * @param conn The database connection. + * @param tapeFileSearchCriteria Search criteria containing a list of disk file IDs (fxid). + * @return Name of the temporary table + */ + virtual std::string createAndPopulateTempTableFxid(rdbms::Conn &conn, const TapeFileSearchCriteria &tapeFileSearchCriteria) const = 0; + /** * Returns a unique archive ID that can be used by a new archive file within * the catalogue. diff --git a/catalogue/RdbmsCatalogueGetArchiveFilesForRepackItor.hpp b/catalogue/RdbmsCatalogueGetArchiveFilesForRepackItor.hpp index c32c7aa474c2aea1e811e51d006e74d6e7532cbe..2ca9965f7d14c633b1a3da04897d0fee6612fe02 100644 --- a/catalogue/RdbmsCatalogueGetArchiveFilesForRepackItor.hpp +++ b/catalogue/RdbmsCatalogueGetArchiveFilesForRepackItor.hpp @@ -20,7 +20,6 @@ #include "catalogue/ArchiveFileBuilder.hpp" #include "catalogue/ArchiveFileItorImpl.hpp" -#include "catalogue/TapeFileSearchCriteria.hpp" #include "common/log/Logger.hpp" #include "rdbms/ConnPool.hpp" #include "rdbms/Stmt.hpp" diff --git a/catalogue/RdbmsCatalogueGetArchiveFilesItor.cpp b/catalogue/RdbmsCatalogueGetArchiveFilesItor.cpp index 0ce1898e5fbe24baa6da19a17e2e679487d787d0..af7debcae983250e70e0ea9ff7cd786b13d99888 100644 --- a/catalogue/RdbmsCatalogueGetArchiveFilesItor.cpp +++ b/catalogue/RdbmsCatalogueGetArchiveFilesItor.cpp @@ -78,14 +78,16 @@ namespace { //------------------------------------------------------------------------------ RdbmsCatalogueGetArchiveFilesItor::RdbmsCatalogueGetArchiveFilesItor( log::Logger &log, - rdbms::ConnPool &connPool, - const TapeFileSearchCriteria &searchCriteria): + rdbms::Conn conn, + const TapeFileSearchCriteria &searchCriteria, + const std::string &tempDiskFxidsTableName) : m_log(log), - m_connPool(connPool), m_searchCriteria(searchCriteria), m_rsetIsEmpty(true), m_hasMoreHasBeenCalled(false), - m_archiveFileBuilder(log) { + m_conn(std::move(conn)), + m_archiveFileBuilder(log) +{ try { std::string sql = "SELECT " @@ -120,16 +122,13 @@ RdbmsCatalogueGetArchiveFilesItor::RdbmsCatalogueGetArchiveFilesItor( "INNER JOIN TAPE_POOL ON " "TAPE.TAPE_POOL_ID = TAPE_POOL.TAPE_POOL_ID"; + const bool hideSuperseded = searchCriteria.showSuperseded ? !*searchCriteria.showSuperseded : false; const bool thereIsAtLeastOneSearchCriteria = searchCriteria.archiveFileId || searchCriteria.diskInstance || - searchCriteria.diskFileId || - searchCriteria.diskFileOwnerUid || - searchCriteria.diskFileGid || - searchCriteria.storageClass || searchCriteria.vid || - searchCriteria.tapeFileCopyNb || - searchCriteria.tapePool; + searchCriteria.diskFileIds || + hideSuperseded; if(thereIsAtLeastOneSearchCriteria) { sql += " WHERE "; @@ -146,50 +145,29 @@ RdbmsCatalogueGetArchiveFilesItor::RdbmsCatalogueGetArchiveFilesItor( sql += "ARCHIVE_FILE.DISK_INSTANCE_NAME = :DISK_INSTANCE_NAME"; addedAWhereConstraint = true; } - if(searchCriteria.diskFileId) { - if(addedAWhereConstraint) sql += " AND "; - sql += "ARCHIVE_FILE.DISK_FILE_ID = :DISK_FILE_ID"; - addedAWhereConstraint = true; - } - if(searchCriteria.diskFileOwnerUid) { - if(addedAWhereConstraint) sql += " AND "; - sql += "ARCHIVE_FILE.DISK_FILE_UID = :DISK_FILE_UID"; - addedAWhereConstraint = true; - } - if(searchCriteria.diskFileGid) { - if(addedAWhereConstraint) sql += " AND "; - sql += "ARCHIVE_FILE.DISK_FILE_GID = :DISK_FILE_GID"; - addedAWhereConstraint = true; - } - if(searchCriteria.storageClass) { - if(addedAWhereConstraint) sql += " AND "; - sql += "STORAGE_CLASS.STORAGE_CLASS_NAME = :STORAGE_CLASS_NAME"; - addedAWhereConstraint = true; - } if(searchCriteria.vid) { if(addedAWhereConstraint) sql += " AND "; sql += "TAPE_FILE.VID = :VID"; addedAWhereConstraint = true; } - if(searchCriteria.tapeFileCopyNb) { + if(searchCriteria.diskFileIds) { if(addedAWhereConstraint) sql += " AND "; - sql += "TAPE_FILE.COPY_NB = :TAPE_FILE_COPY_NB"; + sql += "ARCHIVE_FILE.DISK_FILE_ID IN (SELECT DISK_FILE_ID FROM " + tempDiskFxidsTableName + ")"; addedAWhereConstraint = true; } - if(searchCriteria.tapePool) { + if(hideSuperseded) { if(addedAWhereConstraint) sql += " AND "; - sql += "TAPE_POOL.TAPE_POOL_NAME = :TAPE_POOL_NAME"; + sql += "TAPE_FILE.SUPERSEDED_BY_VID IS NULL"; + addedAWhereConstraint = true; } - // Order by FSEQ if we are listing the contents of a tape, else order by - // archive file ID + // Order by FSEQ if we are listing the contents of a tape, else order by archive file ID if(searchCriteria.vid) { sql += " ORDER BY FSEQ"; } else { sql += " ORDER BY ARCHIVE_FILE_ID, COPY_NB"; } - m_conn = connPool.getConn(); m_stmt = m_conn.createStmt(sql); if(searchCriteria.archiveFileId) { m_stmt.bindUint64(":ARCHIVE_FILE_ID", searchCriteria.archiveFileId.value()); @@ -197,29 +175,10 @@ RdbmsCatalogueGetArchiveFilesItor::RdbmsCatalogueGetArchiveFilesItor( if(searchCriteria.diskInstance) { m_stmt.bindString(":DISK_INSTANCE_NAME", searchCriteria.diskInstance.value()); } - if(searchCriteria.diskFileId) { - m_stmt.bindString(":DISK_FILE_ID", searchCriteria.diskFileId.value()); - } - if(searchCriteria.diskFileOwnerUid) { - m_stmt.bindUint64(":DISK_FILE_UID", searchCriteria.diskFileOwnerUid.value()); - } - if(searchCriteria.diskFileGid) { - m_stmt.bindUint64(":DISK_FILE_GID", searchCriteria.diskFileGid.value()); - } - if(searchCriteria.storageClass) { - m_stmt.bindString(":STORAGE_CLASS_NAME", searchCriteria.storageClass.value()); - } if(searchCriteria.vid) { m_stmt.bindString(":VID", searchCriteria.vid.value()); } - if(searchCriteria.tapeFileCopyNb) { - m_stmt.bindUint64(":TAPE_FILE_COPY_NB", searchCriteria.tapeFileCopyNb.value()); - } - if(searchCriteria.tapePool) { - m_stmt.bindString(":TAPE_POOL_NAME", searchCriteria.tapePool.value()); - } m_rset = m_stmt.executeQuery(); - { log::LogContext lc(m_log); lc.log(log::INFO, "RdbmsCatalogueGetArchiveFilesItor - immediately after m_stmt.executeQuery()"); diff --git a/catalogue/RdbmsCatalogueGetArchiveFilesItor.hpp b/catalogue/RdbmsCatalogueGetArchiveFilesItor.hpp index 91c39ef2f76fa51919d4690a01e664097b66f011..018895fa61fecf14ec6753bf4a84d013b266cd10 100644 --- a/catalogue/RdbmsCatalogueGetArchiveFilesItor.hpp +++ b/catalogue/RdbmsCatalogueGetArchiveFilesItor.hpp @@ -40,14 +40,15 @@ public: * Constructor. * * @param log Object representing the API to the CTA logging system. - * @param connPool The database connection pool. + * @param conn The database connection. * @param searchCriteria The search criteria to be used when listing archive * files. */ RdbmsCatalogueGetArchiveFilesItor( log::Logger &log, - rdbms::ConnPool &connPool, - const TapeFileSearchCriteria &searchCriteria); + rdbms::Conn conn, + const TapeFileSearchCriteria &searchCriteria, + const std::string &tempDiskFxidsTableName); /** * Destructor. @@ -71,11 +72,6 @@ private: */ log::Logger &m_log; - /** - * The database connection pool. - */ - rdbms::ConnPool &m_connPool; - /** * The search criteria to be used when listing archive files. */ diff --git a/catalogue/RdbmsCatalogueGetDeletedArchiveFilesItor.cpp b/catalogue/RdbmsCatalogueGetDeletedArchiveFilesItor.cpp index 1862423ed487724c8ff16fbdafae5f884f77b605..9e7ec4c945371f769b0a5ed0832e12908adbebd5 100644 --- a/catalogue/RdbmsCatalogueGetDeletedArchiveFilesItor.cpp +++ b/catalogue/RdbmsCatalogueGetDeletedArchiveFilesItor.cpp @@ -122,16 +122,13 @@ RdbmsCatalogueGetDeletedArchiveFilesItor::RdbmsCatalogueGetDeletedArchiveFilesIt "INNER JOIN TAPE_POOL ON " "TAPE.TAPE_POOL_ID = TAPE_POOL.TAPE_POOL_ID"; + const bool hideSuperseded = searchCriteria.showSuperseded ? !*searchCriteria.showSuperseded : false; const bool thereIsAtLeastOneSearchCriteria = searchCriteria.archiveFileId || searchCriteria.diskInstance || - searchCriteria.diskFileId || - searchCriteria.diskFileOwnerUid || - searchCriteria.diskFileGid || - searchCriteria.storageClass || searchCriteria.vid || - searchCriteria.tapeFileCopyNb || - searchCriteria.tapePool; + searchCriteria.diskFileIds || + hideSuperseded; if(thereIsAtLeastOneSearchCriteria) { sql += " WHERE "; @@ -140,47 +137,34 @@ RdbmsCatalogueGetDeletedArchiveFilesItor::RdbmsCatalogueGetDeletedArchiveFilesIt bool addedAWhereConstraint = false; if(searchCriteria.archiveFileId) { - sql += " ARCHIVE_FILE_RECYCLE_BIN.ARCHIVE_FILE_ID = :ARCHIVE_FILE_ID"; + sql += " ARCHIVE_FILE.ARCHIVE_FILE_ID = :ARCHIVE_FILE_ID"; addedAWhereConstraint = true; } if(searchCriteria.diskInstance) { if(addedAWhereConstraint) sql += " AND "; - sql += "ARCHIVE_FILE_RECYCLE_BIN.DISK_INSTANCE_NAME = :DISK_INSTANCE_NAME"; - addedAWhereConstraint = true; - } - if(searchCriteria.diskFileId) { - if(addedAWhereConstraint) sql += " AND "; - sql += "ARCHIVE_FILE_RECYCLE_BIN.DISK_FILE_ID = :DISK_FILE_ID"; - addedAWhereConstraint = true; - } - if(searchCriteria.diskFileOwnerUid) { - if(addedAWhereConstraint) sql += " AND "; - sql += "ARCHIVE_FILE_RECYCLE_BIN.DISK_FILE_UID = :DISK_FILE_UID"; - addedAWhereConstraint = true; - } - if(searchCriteria.diskFileGid) { - if(addedAWhereConstraint) sql += " AND "; - sql += "ARCHIVE_FILE_RECYCLE_BIN.DISK_FILE_GID = :DISK_FILE_GID"; - addedAWhereConstraint = true; - } - if(searchCriteria.storageClass) { - if(addedAWhereConstraint) sql += " AND "; - sql += "STORAGE_CLASS.STORAGE_CLASS_NAME = :STORAGE_CLASS_NAME"; + sql += "ARCHIVE_FILE.DISK_INSTANCE_NAME = :DISK_INSTANCE_NAME"; addedAWhereConstraint = true; } if(searchCriteria.vid) { if(addedAWhereConstraint) sql += " AND "; - sql += "TAPE_FILE_RECYCLE_BIN.VID = :VID"; + sql += "TAPE_FILE.VID = :VID"; addedAWhereConstraint = true; } - if(searchCriteria.tapeFileCopyNb) { + if(searchCriteria.diskFileIds) { if(addedAWhereConstraint) sql += " AND "; - sql += "TAPE_FILE_RECYCLE_BIN.COPY_NB = :TAPE_FILE_COPY_NB"; + sql += "ARCHIVE_FILE.DISK_FILE_ID IN "; + char delim = '('; + for(auto &diskFileId : searchCriteria.diskFileIds.value()) { + sql += delim + diskFileId; + delim = ','; + } + sql += ')'; addedAWhereConstraint = true; } - if(searchCriteria.tapePool) { + if(hideSuperseded) { if(addedAWhereConstraint) sql += " AND "; - sql += "TAPE_POOL.TAPE_POOL_NAME = :TAPE_POOL_NAME"; + sql += "TAPE_FILE.SUPERSEDED_BY_VID != ''"; + addedAWhereConstraint = true; } // Order by FSEQ if we are listing the contents of a tape, else order by @@ -199,27 +183,9 @@ RdbmsCatalogueGetDeletedArchiveFilesItor::RdbmsCatalogueGetDeletedArchiveFilesIt if(searchCriteria.diskInstance) { m_stmt.bindString(":DISK_INSTANCE_NAME", searchCriteria.diskInstance.value()); } - if(searchCriteria.diskFileId) { - m_stmt.bindString(":DISK_FILE_ID", searchCriteria.diskFileId.value()); - } - if(searchCriteria.diskFileOwnerUid) { - m_stmt.bindUint64(":DISK_FILE_UID", searchCriteria.diskFileOwnerUid.value()); - } - if(searchCriteria.diskFileGid) { - m_stmt.bindUint64(":DISK_FILE_GID", searchCriteria.diskFileGid.value()); - } - if(searchCriteria.storageClass) { - m_stmt.bindString(":STORAGE_CLASS_NAME", searchCriteria.storageClass.value()); - } if(searchCriteria.vid) { m_stmt.bindString(":VID", searchCriteria.vid.value()); } - if(searchCriteria.tapeFileCopyNb) { - m_stmt.bindUint64(":TAPE_FILE_COPY_NB", searchCriteria.tapeFileCopyNb.value()); - } - if(searchCriteria.tapePool) { - m_stmt.bindString(":TAPE_POOL_NAME", searchCriteria.tapePool.value()); - } m_rset = m_stmt.executeQuery(); { diff --git a/catalogue/SqliteCatalogue.cpp b/catalogue/SqliteCatalogue.cpp index ef10de882691ba7bfa0440780a587fa1101556cb..c59abad6d2b6ee8c92f82f26cce59d14c1f9f0a4 100644 --- a/catalogue/SqliteCatalogue.cpp +++ b/catalogue/SqliteCatalogue.cpp @@ -193,6 +193,31 @@ void SqliteCatalogue::DO_NOT_USE_deleteArchiveFile_DO_NOT_USE(const std::string } } +//------------------------------------------------------------------------------ +// createAndPopulateTempTableFxid +//------------------------------------------------------------------------------ +std::string SqliteCatalogue::createAndPopulateTempTableFxid(rdbms::Conn &conn, const TapeFileSearchCriteria &tapeFileSearchCriteria) const { + const std::string tempTableName = "TEMP.DISK_FXIDS"; + + if(tapeFileSearchCriteria.diskFileIds) { + try { + // Drop any prexisting temporary table and create a new one + conn.executeNonQuery("DROP TABLE IF EXISTS " + tempTableName); + conn.executeNonQuery("CREATE TEMPORARY TABLE " + tempTableName + "(DISK_FILE_ID TEXT)"); + + auto stmt = conn.createStmt("INSERT INTO " + tempTableName + " VALUES(:DISK_FILE_ID)"); + for(auto &diskFileId : tapeFileSearchCriteria.diskFileIds.value()) { + stmt.bindString(":DISK_FILE_ID", diskFileId); + stmt.executeNonQuery(); + } + } catch(exception::Exception &ex) { + ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str()); + throw; + } + } + return tempTableName; +} + //------------------------------------------------------------------------------ // getNextArchiveFileId //------------------------------------------------------------------------------ diff --git a/catalogue/SqliteCatalogue.hpp b/catalogue/SqliteCatalogue.hpp index af0848cf1f332dd413432165a3a876d7d5f5a237..66c6f8577f9ef1199b5f402d2c7d6f88b2c911ba 100644 --- a/catalogue/SqliteCatalogue.hpp +++ b/catalogue/SqliteCatalogue.hpp @@ -83,6 +83,15 @@ public: protected: + /** + * Creates a temporary table from the list of disk file IDs provided in the search criteria. + * + * @param conn The database connection. + * @param tapeFileSearchCriteria Search criteria containing a list of disk file IDs (fxid). + * @return Name of the temporary table + */ + std::string createAndPopulateTempTableFxid(rdbms::Conn &conn, const TapeFileSearchCriteria &tapeFileSearchCriteria) const override; + /** * Returns a unique archive ID that can be used by a new archive file within * the catalogue. diff --git a/catalogue/TapeFileSearchCriteria.hpp b/catalogue/TapeFileSearchCriteria.hpp index 3ce165e1635592256fa49a46f4c687b718501b47..58915f12bdccf1a0ab75a90ca726713575427eaa 100644 --- a/catalogue/TapeFileSearchCriteria.hpp +++ b/catalogue/TapeFileSearchCriteria.hpp @@ -19,6 +19,7 @@ #pragma once #include <string> +#include <vector> #include "common/optional.hpp" @@ -27,15 +28,11 @@ namespace catalogue { /** * The collection of criteria used to select a set of tape files. - * - * An tape file is selected if it meets all of the specified criteria. - * + * A tape file is selected if it meets all of the specified criteria. * A criterion is only considered specified if it has been set. - * * Please note that no wild cards, for example '*' or '%', are supported. */ struct TapeFileSearchCriteria { - /** * The unique identifier of an archive file. */ @@ -46,44 +43,23 @@ struct TapeFileSearchCriteria { */ optional<std::string> diskInstance; - /** - * The unique identifier of a disk file within its disk instance. - * - * The combination of diskInstance and diskFileId is unique across all disk - * instances. - */ - optional<std::string> diskFileId; - - /** - * The owner of a file within its disk instance. - */ - optional<uint64_t> diskFileOwnerUid; - - /** - * The group of a file within its disk instance. - */ - optional<uint64_t> diskFileGid; - - /** - * The storage class name of the file. - */ - optional<std::string> storageClass; - /** * The volume identifier of a tape. */ optional<std::string> vid; /** - * The copy number of a tape file. + * List of disk file IDs. + * + * These are given as a list of strings in DECIMAL format. EOS provides the fxids in hex format. The parsing and + * conversion into decimal is done in the cta-admin client, ready to be built into a SQL query string. */ - optional<uint64_t> tapeFileCopyNb; + optional<std::vector<std::string>> diskFileIds; /** - * The name of a tape pool. + * Include superseded files in the output? */ - optional<std::string> tapePool; - + optional<bool> showSuperseded; }; // struct TapeFileSearchCriteria } // namespace catalogue diff --git a/cmdline/CtaAdminCmdParse.hpp b/cmdline/CtaAdminCmdParse.hpp index 414195d3d82ee7536e14d790d2ab2102c14c0e2a..4affaf78b9327c41f3359b6410e30c35159efdcc 100644 --- a/cmdline/CtaAdminCmdParse.hpp +++ b/cmdline/CtaAdminCmdParse.hpp @@ -2,7 +2,7 @@ * @project The CERN Tape Archive (CTA) * @brief Definitions for parsing the options of the CTA Admin command-line tool * @description CTA Admin command using Google Protocol Buffers and XRootD SSI transport - * @copyright Copyright 2017 CERN + * @copyright Copyright 2020 CERN * @license This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or @@ -181,8 +181,6 @@ private: const cmdLookup_t cmdLookup = { { "admin", AdminCmd::CMD_ADMIN }, { "ad", AdminCmd::CMD_ADMIN }, - { "archivefile", AdminCmd::CMD_ARCHIVEFILE }, - { "af", AdminCmd::CMD_ARCHIVEFILE }, { "archiveroute", AdminCmd::CMD_ARCHIVEROUTE }, { "ar", AdminCmd::CMD_ARCHIVEROUTE }, { "drive", AdminCmd::CMD_DRIVE }, @@ -191,10 +189,6 @@ const cmdLookup_t cmdLookup = { { "fr", AdminCmd::CMD_FAILEDREQUEST }, { "groupmountrule", AdminCmd::CMD_GROUPMOUNTRULE }, { "gmr", AdminCmd::CMD_GROUPMOUNTRULE }, - { "listpendingarchives", AdminCmd::CMD_LISTPENDINGARCHIVES }, - { "lpa", AdminCmd::CMD_LISTPENDINGARCHIVES }, - { "listpendingretrieves", AdminCmd::CMD_LISTPENDINGRETRIEVES }, - { "lpr", AdminCmd::CMD_LISTPENDINGRETRIEVES }, { "logicallibrary", AdminCmd::CMD_LOGICALLIBRARY }, { "ll", AdminCmd::CMD_LOGICALLIBRARY }, { "mediatype", AdminCmd::CMD_MEDIATYPE }, @@ -260,15 +254,14 @@ const std::map<std::string, OptionBoolean::Key> boolOptions = { // hasOption options { "--checkchecksum", OptionBoolean::CHECK_CHECKSUM }, - { "--disabledtape", OptionBoolean::DISABLED }, - { "--extended", OptionBoolean::EXTENDED }, + { "--disabledtape", OptionBoolean::DISABLED }, { "--justarchive", OptionBoolean::JUSTARCHIVE }, { "--justmove", OptionBoolean::JUSTMOVE }, { "--justaddcopies", OptionBoolean::JUSTADDCOPIES }, { "--justretrieve", OptionBoolean::JUSTRETRIEVE }, { "--log", OptionBoolean::SHOW_LOG_ENTRIES }, { "--lookupnamespace", OptionBoolean::LOOKUP_NAMESPACE }, - { "--summary", OptionBoolean::SUMMARY }, + { "--showsuperseded", OptionBoolean::SHOW_SUPERSEDED }, { "--no-recall", OptionBoolean::NO_RECALL} }; @@ -316,6 +309,7 @@ const std::map<std::string, OptionString::Key> strOptions = { { "--diskid", OptionString::DISKID }, { "--drive", OptionString::DRIVE }, { "--encryptionkeyname", OptionString::ENCRYPTION_KEY_NAME }, + { "--fxid", OptionString::FXID }, { "--file", OptionString::FILENAME }, { "--hostname", OptionString::HOSTNAME }, { "--input", OptionString::INPUT }, @@ -344,7 +338,7 @@ const std::map<std::string, OptionString::Key> strOptions = { * Map string list options to Protocol Buffer enum values */ const std::map<std::string, OptionStrList::Key> strListOptions = { - { "--fidfile", OptionStrList::FILE_ID }, + { "--fxidfile", OptionStrList::FILE_ID }, { "--vidfile", OptionStrList::VID } }; @@ -355,7 +349,6 @@ const std::map<std::string, OptionStrList::Key> strListOptions = { */ const std::map<AdminCmd::Cmd, CmdHelp> cmdHelp = { { AdminCmd::CMD_ADMIN, { "admin", "ad", { "add", "ch", "rm", "ls" } }}, - { AdminCmd::CMD_ARCHIVEFILE, { "archivefile", "af", { "ls" } }}, { AdminCmd::CMD_ARCHIVEROUTE, { "archiveroute", "ar", { "add", "ch", "rm", "ls" } }}, { AdminCmd::CMD_DRIVE, { "drive", "dr", { "up", "down", "ls", "ch", "rm" }, "\n This is a synchronous command that sets and reads back the state of one or\n" @@ -365,8 +358,6 @@ const std::map<AdminCmd::Cmd, CmdHelp> cmdHelp = { }}, { AdminCmd::CMD_FAILEDREQUEST, { "failedrequest", "fr", { "ls", "show", "retry", "rm" } }}, { AdminCmd::CMD_GROUPMOUNTRULE, { "groupmountrule", "gmr", { "add", "ch", "rm", "ls" } }}, - { AdminCmd::CMD_LISTPENDINGARCHIVES, { "listpendingarchives", "lpa", { } }}, - { AdminCmd::CMD_LISTPENDINGRETRIEVES, { "listpendingretrieves", "lpr", { } }}, { AdminCmd::CMD_LOGICALLIBRARY, { "logicallibrary", "ll", { "add", "ch", "rm", "ls" } }}, { AdminCmd::CMD_MEDIATYPE, { "mediatype", "mt", { "add", "ch", "rm", "ls" } }}, { AdminCmd::CMD_MOUNTPOLICY, { "mountpolicy", "mp", { "add", "ch", "rm", "ls" } }}, @@ -389,7 +380,11 @@ const std::map<AdminCmd::Cmd, CmdHelp> cmdHelp = { { AdminCmd::CMD_SHOWQUEUES, { "showqueues", "sq", { } }}, { AdminCmd::CMD_STORAGECLASS, { "storageclass", "sc", { "add", "ch", "rm", "ls" } }}, { AdminCmd::CMD_TAPE, { "tape", "ta", { "add", "ch", "rm", "reclaim", "ls", "label" } }}, - { AdminCmd::CMD_TAPEFILE, { "tapefile", "tf", { "ls" } }}, + { AdminCmd::CMD_TAPEFILE, { "tapefile", "tf", { "ls" }, + " Tape files can be listed by VID or by EOS disk instance + EOS disk file ID.\n" + " Disk file IDs should be provided in hexadecimal (fxid). The --fxidfile option\n" + " takes a file in the same format as the output of 'eos find --fid <path>'\n\n" + }}, { AdminCmd::CMD_TAPEPOOL, { "tapepool", "tp", { "add", "ch", "rm", "ls" } }}, { AdminCmd::CMD_DISKSYSTEM, { "disksystem", "ds", { "add", "ch", "rm", "ls" }, "\n This command allows to manage disk systems (useful for the backpressure).\n" @@ -435,17 +430,16 @@ const Option opt_drivename { Option::OPT_STR, "--drive", const Option opt_drivename_cmd { Option::OPT_CMD, "--drive", "", "<drive_name>" }; const Option opt_encrypted { Option::OPT_BOOL, "--encrypted", "-e", " <\"true\" or \"false\">" }; const Option opt_encryptionkeyname { Option::OPT_STR, "--encryptionkeyname", "-k", " <encryption_key_name>" }; -const Option opt_extended { Option::OPT_FLAG, "--extended", "-x", "" }; -const Option opt_fidfile { Option::OPT_STR_LIST, "--fidfile", "-i", " <filename>" }; +const Option opt_fid { Option::OPT_STR, "--fxid", "-f", " <eos_fxid>" }; +const Option opt_fidfile { Option::OPT_STR_LIST, "--fxidfile", "-F", " <filename>" }; const Option opt_filename { Option::OPT_STR, "--file", "-f", " <filename>" }; const Option opt_firstfseq { Option::OPT_UINT, "--firstfseq", "-f", " <first_fseq>" }; const Option opt_force { Option::OPT_BOOL, "--force", "-f", " <\"true\" or \"false\">" }; const Option opt_force_flag { Option::OPT_FLAG, "--force", "-f", "" }; const Option opt_gid { Option::OPT_UINT, "--gid", "-g", " <group_id>" }; -const Option opt_hostname_alias { Option::OPT_STR, "--name", "-n", " <host_name>", - "--hostname" }; +const Option opt_hostname_alias { Option::OPT_STR, "--name", "-n", " <host_name>", "--hostname" }; const Option opt_input { Option::OPT_STR, "--input", "-i", " <\"zero\" or \"urandom\">" }; -const Option opt_instance { Option::OPT_STR, "--instance", "-i", " <instance_name>" }; +const Option opt_instance { Option::OPT_STR, "--instance", "-i", " <disk_instance>" }; const Option opt_justarchive { Option::OPT_FLAG, "--justarchive", "-a", "" }; const Option opt_justmove { Option::OPT_FLAG, "--justmove", "-m", "" }; const Option opt_justaddcopies { Option::OPT_FLAG, "--justaddcopies", "-a", "" }; @@ -453,60 +447,53 @@ const Option opt_justretrieve { Option::OPT_FLAG, "--justretrieve", const Option opt_lastfseq { Option::OPT_UINT, "--lastfseq", "-l", " <last_fseq>" }; const Option opt_log { Option::OPT_FLAG, "--log", "-l", "" }; const Option opt_logicallibrary { Option::OPT_STR, "--logicallibrary", "-l", " <logical_library_name>" }; -const Option opt_logicallibrary_alias { Option::OPT_STR, "--name", "-n", " <logical_library_name>", - "--logicallibrary" }; +const Option opt_logicallibrary_alias { Option::OPT_STR, "--name", "-n", " <logical_library_name>", "--logicallibrary" }; const Option opt_lookupns { Option::OPT_FLAG, "--lookupnamespace", "-l", "" }; const Option opt_maxdrivesallowed { Option::OPT_UINT, "--maxdrivesallowed", "-d", " <max_drives_allowed>" }; -const Option opt_maxlpos { Option::OPT_UINT, "--maxlpos", "-maxl", " <maximum_longitudinal_position>" }; +const Option opt_maxlpos { Option::OPT_UINT, "--maxlpos", "-maxl", " <maximum_longitudinal_position>" }; const Option opt_mediatype { Option::OPT_STR, "--mediatype", "--mt", " <media_type_name>" }; -const Option opt_mediatype_alias { Option::OPT_STR, "--name", "-n", " <media_type_name>", - "--mediatype" }; +const Option opt_mediatype_alias { Option::OPT_STR, "--name", "-n", " <media_type_name>", "--mediatype" }; const Option opt_minarchiverequestage { Option::OPT_UINT, "--minarchiverequestage", "--aa", " <min_request_age>" }; -const Option opt_minlpos { Option::OPT_UINT, "--minlpos", "-minl", " <minimum_longitudinal_position>" }; +const Option opt_minlpos { Option::OPT_UINT, "--minlpos", "-minl", " <minimum_longitudinal_position>" }; const Option opt_minretrieverequestage{ Option::OPT_UINT, "--minretrieverequestage", "--ra", " <min_request_age>" }; const Option opt_mountpolicy { Option::OPT_STR, "--mountpolicy", "-u", " <mount_policy_name>" }; -const Option opt_mountpolicy_alias { Option::OPT_STR, "--name", "-n", " <mount_policy_name>", - "--mountpolicy" }; +const Option opt_mountpolicy_alias { Option::OPT_STR, "--name", "-n", " <mount_policy_name>", "--mountpolicy" }; const Option opt_number_of_files { Option::OPT_UINT, "--nbfiles", "-n", " <number_of_files_per_tape>" }; -const Option opt_number_of_files_alias{ Option::OPT_UINT, "--number", "-n", " <number_of_files>", - "--nbfiles" }; +const Option opt_number_of_files_alias{ Option::OPT_UINT, "--number", "-n", " <number_of_files>", "--nbfiles" }; const Option opt_number_of_wraps { Option::OPT_UINT, "--nbwraps", "-w", " <number_of_wraps>" }; const Option opt_output { Option::OPT_STR, "--output", "-o", " <\"null\" or output_dir>" }; const Option opt_owner_uid { Option::OPT_UINT, "--uid", "-u", " <owner_uid>" }; const Option opt_partialfiles { Option::OPT_UINT, "--partial", "-p", " <number_of_files_per_tape>" }; const Option opt_partialtapes { Option::OPT_UINT, "--partialtapesnumber", "-p", " <number_of_partial_tapes>" }; const Option opt_path { Option::OPT_STR, "--path", "-p", " <full_path>" }; -const Option opt_primarydensitycode { Option::OPT_UINT, "--primarydensitycode", "-p", " <primary_density_code>" }; +const Option opt_primarydensitycode { Option::OPT_UINT, "--primarydensitycode", "-p", " <primary_density_code>" }; const Option opt_retrievepriority { Option::OPT_UINT, "--retrievepriority", "--rp", " <priority_value>" }; const Option opt_secondarydensitycode { Option::OPT_UINT, "--secondarydensitycode", "-s", " <secondary_density_code>" }; const Option opt_size { Option::OPT_UINT, "--size", "-s", " <file_size>" }; const Option opt_storageclass { Option::OPT_STR, "--storageclass", "-s", " <storage_class_name>" }; -const Option opt_storageclass_alias { Option::OPT_STR, "--name", "-n", " <storage_class_name>", - "--storageclass" }; +const Option opt_storageclass_alias { Option::OPT_STR, "--name", "-n", " <storage_class_name>", "--storageclass" }; const Option opt_summary { Option::OPT_FLAG, "--summary", "-S", "" }; const Option opt_supply { Option::OPT_STR, "--supply", "-s", " <supply_value>" }; const Option opt_tapepool { Option::OPT_STR, "--tapepool", "-t", " <tapepool_name>" }; -const Option opt_tapepool_alias { Option::OPT_STR, "--name", "-n", " <tapepool_name>", - "--tapepool" }; +const Option opt_tapepool_alias { Option::OPT_STR, "--name", "-n", " <tapepool_name>", "--tapepool" }; const Option opt_username { Option::OPT_STR, "--username", "-u", " <user_name>" }; -const Option opt_username_alias { Option::OPT_STR, "--name", "-n", " <user_name>", - "--username" }; +const Option opt_username_alias { Option::OPT_STR, "--name", "-n", " <user_name>", "--username" }; const Option opt_vendor { Option::OPT_STR, "--vendor", "--ve", " <vendor>" }; const Option opt_vid { Option::OPT_STR, "--vid", "-v", " <vid>" }; const Option opt_vo { Option::OPT_STR, "--vo", "--vo", " <vo>" }; const Option opt_vidfile { Option::OPT_STR_LIST, "--vidfile", "-f", " <filename>" }; const Option opt_full { Option::OPT_BOOL, "--full", "-f", " <\"true\" or \"false\">" }; const Option opt_readonly { Option::OPT_BOOL, "--readonly", "-r", " <\"true\" or \"false\">" }; -const Option opt_disabled_tape { Option::OPT_FLAG, "--disabledtape", "-d", ""}; - -const Option opt_disksystem { Option::OPT_STR, "--disksystem", "-n", " <disk_system_name>" }; -const Option opt_file_regexp { Option::OPT_STR, "--fileregexp", "-r", " <file_regexp>" }; -const Option opt_free_space_query_url { Option::OPT_STR, "--freespacequeryurl", "-u", " <free_space_query_url>" }; -const Option opt_refresh_interval { Option::OPT_UINT, "--refreshinterval", "-i", " <refresh_intreval>" }; -const Option opt_targeted_free_space { Option::OPT_UINT, "--targetedfreespace", "-f", " <targeted_free_space>" }; -const Option opt_sleep_time { Option::OPT_UINT, "--sleeptime", "-s", " <sleep time in s>" }; -const Option opt_reason { Option::OPT_STR, "--reason", "-r", " <reason_status_change>" }; -const Option opt_no_recall { Option::OPT_FLAG, "--no-recall", "-nr", "" }; +const Option opt_disabled_tape { Option::OPT_FLAG, "--disabledtape", "-d", "" }; +const Option opt_disksystem { Option::OPT_STR, "--disksystem", "-n", " <disk_system_name>" }; +const Option opt_file_regexp { Option::OPT_STR, "--fileregexp", "-r", " <file_regexp>" }; +const Option opt_free_space_query_url { Option::OPT_STR, "--freespacequeryurl", "-u", " <free_space_query_url>" }; +const Option opt_refresh_interval { Option::OPT_UINT, "--refreshinterval", "-i", " <refresh_intreval>" }; +const Option opt_targeted_free_space { Option::OPT_UINT, "--targetedfreespace", "-f", " <targeted_free_space>" }; +const Option opt_sleep_time { Option::OPT_UINT, "--sleeptime", "-s", " <sleep time in s>" }; +const Option opt_reason { Option::OPT_STR, "--reason", "-r", " <reason_status_change>" }; +const Option opt_show_superseded { Option::OPT_FLAG, "--showsuperseded", "-s", "" }; +const Option opt_no_recall { Option::OPT_FLAG, "--no-recall", "-nr", "" }; /*! * Map valid options to commands @@ -517,12 +504,6 @@ const std::map<cmd_key_t, cmd_val_t> cmdOptions = { {{ AdminCmd::CMD_ADMIN, AdminCmd::SUBCMD_RM }, { opt_username }}, {{ AdminCmd::CMD_ADMIN, AdminCmd::SUBCMD_LS }, { }}, /*----------------------------------------------------------------------------------------------------*/ - {{ AdminCmd::CMD_ARCHIVEFILE, AdminCmd::SUBCMD_LS }, - { opt_archivefileid.optional(), opt_diskid.optional(), opt_copynb.optional(), - opt_vid.optional(), opt_tapepool.optional(), opt_owner_uid.optional(), opt_gid.optional(), - opt_storageclass.optional(), opt_path.optional(), opt_instance.optional(), opt_all.optional(), - opt_summary.optional() }}, - /*----------------------------------------------------------------------------------------------------*/ {{ AdminCmd::CMD_ARCHIVEROUTE, AdminCmd::SUBCMD_ADD }, { opt_storageclass, opt_copynb, opt_tapepool, opt_comment }}, {{ AdminCmd::CMD_ARCHIVEROUTE, AdminCmd::SUBCMD_CH }, @@ -550,12 +531,6 @@ const std::map<cmd_key_t, cmd_val_t> cmdOptions = { {{ AdminCmd::CMD_GROUPMOUNTRULE, AdminCmd::SUBCMD_RM }, { opt_instance, opt_username_alias }}, {{ AdminCmd::CMD_GROUPMOUNTRULE, AdminCmd::SUBCMD_LS }, { }}, /*----------------------------------------------------------------------------------------------------*/ - {{ AdminCmd::CMD_LISTPENDINGARCHIVES, AdminCmd::SUBCMD_NONE }, - { opt_tapepool.optional(), opt_extended.optional() }}, - /*----------------------------------------------------------------------------------------------------*/ - {{ AdminCmd::CMD_LISTPENDINGRETRIEVES, AdminCmd::SUBCMD_NONE }, - { opt_vid.optional(), opt_extended.optional() }}, - /*----------------------------------------------------------------------------------------------------*/ {{ AdminCmd::CMD_LOGICALLIBRARY, AdminCmd::SUBCMD_ADD }, { opt_logicallibrary_alias, opt_disabled.optional(), opt_comment }}, {{ AdminCmd::CMD_LOGICALLIBRARY, AdminCmd::SUBCMD_CH }, @@ -618,7 +593,8 @@ const std::map<cmd_key_t, cmd_val_t> cmdOptions = { {{ AdminCmd::CMD_TAPE, AdminCmd::SUBCMD_LABEL }, { opt_vid, opt_force.optional() }}, /*----------------------------------------------------------------------------------------------------*/ - {{ AdminCmd::CMD_TAPEFILE, AdminCmd::SUBCMD_LS }, { opt_vid, opt_lookupns.optional() }}, + {{ AdminCmd::CMD_TAPEFILE, AdminCmd::SUBCMD_LS }, + { opt_vid.optional(), opt_instance.optional(), opt_fid.optional(), opt_fidfile.optional(), opt_show_superseded.optional(), opt_lookupns.optional() }}, /*----------------------------------------------------------------------------------------------------*/ {{ AdminCmd::CMD_TAPEPOOL, AdminCmd::SUBCMD_ADD }, { opt_tapepool_alias, opt_vo, opt_partialtapes, opt_encrypted, opt_supply.optional(), opt_comment }}, diff --git a/cmdline/CtaAdminTextFormatter.cpp b/cmdline/CtaAdminTextFormatter.cpp index b26886bc9ba1d2504568738e514c9595f436e654..8b0d17ff1ff7720f37e29a5ab3ef74f9a38f2d45 100644 --- a/cmdline/CtaAdminTextFormatter.cpp +++ b/cmdline/CtaAdminTextFormatter.cpp @@ -893,7 +893,7 @@ void TextFormatter::printTapeFileLsHeader() { "fseq", "block id", "instance", - "disk id", + "disk fxid", "size", "checksum type", "checksum value", @@ -901,8 +901,8 @@ void TextFormatter::printTapeFileLsHeader() { "owner", "group", "creation time", - "sc vid", // superceded - "sc fseq", + "ss vid", // superseded + "ss fseq", "path" ); } @@ -915,9 +915,12 @@ void TextFormatter::print(const TapeFileLsItem &tfls_item) { if(!tfls_item.af().checksum().empty()) { const google::protobuf::EnumDescriptor *descriptor = cta::common::ChecksumBlob::Checksum::Type_descriptor(); - std::string name = descriptor->FindValueByNumber(tfls_item.af().checksum().begin()->type())->name(); - checksumValue = "0x" + tfls_item.af().checksum().begin()->value(); + checksumType = descriptor->FindValueByNumber(tfls_item.af().checksum().begin()->type())->name(); + checksumValue = tfls_item.af().checksum().begin()->value(); } + auto fid = strtol(tfls_item.df().disk_id().c_str(), nullptr, 10); + std::stringstream fxid; + fxid << std::hex << fid; push_back( tfls_item.af().archive_id(), @@ -926,7 +929,7 @@ void TextFormatter::print(const TapeFileLsItem &tfls_item) { tfls_item.tf().f_seq(), tfls_item.tf().block_id(), tfls_item.df().disk_instance(), - tfls_item.df().disk_id(), + fxid.str(), dataSizeToStr(tfls_item.af().size()), checksumType, checksumValue, diff --git a/continuousintegration/orchestration/tests/client_ar.sh b/continuousintegration/orchestration/tests/client_ar.sh index 24f3298a1be07b849cf6000f972d4ba225b2355f..cf253d3c88bb0d30d1df144125200f565637448e 100644 --- a/continuousintegration/orchestration/tests/client_ar.sh +++ b/continuousintegration/orchestration/tests/client_ar.sh @@ -64,7 +64,7 @@ nsls_tapes() # 1. Query EOS namespace to get a list of file IDs # 2. Pipe to "tape ls" to get the list of tapes where those files are archived eos root://${EOSINSTANCE} find --fid ${EOS_DIR} |\ - admin_cta --json tape ls --fidfile /dev/stdin |\ + admin_cta --json tape ls --fxidfile /dev/stdin |\ jq '.[] | .vid' | sed 's/"//g' } diff --git a/cta.spec.in b/cta.spec.in index 650b01c489b390ee3ca618fc7097bba96237f02b..2d0888f128c88f8c2ed8ad530f6915aa596cb4a8 100644 --- a/cta.spec.in +++ b/cta.spec.in @@ -300,7 +300,6 @@ Tools allowing initialization and inspection of the object store. %attr(0755,root,root) %{_bindir}/cta-objectstore-initialize %attr(0755,root,root) %{_bindir}/cta-objectstore-list %attr(0755,root,root) %{_bindir}/cta-objectstore-dump-object -%attr(0755,root,root) %{_bindir}/cta-objectstore-unfollow-agent %attr(0755,root,root) %{_bindir}/cta-objectstore-dereference-removed-queues %attr(0755,root,root) %{_bindir}/cta-objectstore-collect-orphaned-object %attr(0755,root,root) %{_bindir}/cta-objectstore-create-missing-repack-index diff --git a/objectstore/CMakeLists.txt b/objectstore/CMakeLists.txt index a7173f4a10f1fdb341f99f01232f1f1bb09c3987..eadb2459f2072a2817effcf319948ccfe051b1ba 100644 --- a/objectstore/CMakeLists.txt +++ b/objectstore/CMakeLists.txt @@ -142,11 +142,6 @@ set_property(TARGET cta-objectstore-dump-object APPEND PROPERTY INSTALL_RPATH ${ target_link_libraries(cta-objectstore-dump-object ${PROTOBUF3_LIBRARIES} ctaobjectstore ctacommon) -add_executable(cta-objectstore-unfollow-agent cta-objectstore-unfollow-agent.cpp) -set_property(TARGET cta-objectstore-unfollow-agent APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH}) -target_link_libraries(cta-objectstore-unfollow-agent - ${PROTOBUF3_LIBRARIES} ctaobjectstore ctacommon) - add_executable(cta-objectstore-dereference-removed-queues cta-objectstore-dereference-removed-queues.cpp) set_property(TARGET cta-objectstore-dereference-removed-queues APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH}) target_link_libraries(cta-objectstore-dereference-removed-queues @@ -162,6 +157,6 @@ set_property(TARGET cta-objectstore-create-missing-repack-index APPEND PROPERTY target_link_libraries(cta-objectstore-create-missing-repack-index ${PROTOBUF3_LIBRARIES} ctaobjectstore ctacommon) -install(TARGETS cta-objectstore-initialize cta-objectstore-list cta-objectstore-dump-object cta-objectstore-unfollow-agent +install(TARGETS cta-objectstore-initialize cta-objectstore-list cta-objectstore-dump-object cta-objectstore-dereference-removed-queues cta-objectstore-collect-orphaned-object cta-objectstore-create-missing-repack-index DESTINATION usr/bin) diff --git a/objectstore/cta-objectstore-unfollow-agent.cpp b/objectstore/cta-objectstore-unfollow-agent.cpp deleted file mode 100644 index db6a1b2026a60a7c6894873b6e8e9073cda358f6..0000000000000000000000000000000000000000 --- a/objectstore/cta-objectstore-unfollow-agent.cpp +++ /dev/null @@ -1,145 +0,0 @@ -/* - * The CERN Tape Archive (CTA) project - * Copyright (C) 2015 CERN - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - */ - -/** - * This program will create a VFS backend for the object store and populate - * it with the minimum elements (the root entry). The program will then print out - * the path the backend store and exit - */ - -#include "Agent.hpp" -#include "AgentRegister.hpp" -#include "BackendFactory.hpp" -#include "BackendVFS.hpp" -#include "common/Configuration.hpp" -#include "common/log/StdoutLogger.hpp" -#include "common/log/LogContext.hpp" -#include "common/utils/utils.hpp" -#include "RootEntry.hpp" -#include "SerializersExceptions.hpp" - -#include <iostream> -#include <stdexcept> - -int main(int argc, char ** argv) { - try { - cta::log::StdoutLogger logger(cta::utils::getShortHostname(), "cta-objectstore-unfollow-agent"); - std::string agentName; - std::unique_ptr<cta::objectstore::Backend> be; - if (3 == argc) { - be.reset(cta::objectstore::BackendFactory::createBackend(argv[1], logger).release()); - agentName = argv[2]; - } else if (2 == argc) { - cta::common::Configuration m_ctaConf("/etc/cta/cta-objectstore-tools.conf"); - be=std::move(cta::objectstore::BackendFactory::createBackend(m_ctaConf.getConfEntString("ObjectStore", "BackendPath", nullptr), logger)); - agentName = argv[1]; - } else { - throw std::runtime_error("Wrong number of arguments: expected 1 or 2: [objectstoreURL] agentName"); - } - // If the backend is a VFS, make sure we don't delete it on exit. - // If not, nevermind. - try { - dynamic_cast<cta::objectstore::BackendVFS &>(*be).noDeleteOnExit(); - } catch (std::bad_cast &){} - std::cout /* << "Object store path: " << be->getParams()->toURL() - << " agent */<< "name=" << agentName << std::endl; - if (!be->exists(agentName)) { - // Agent does not exist: remove from registry. - cta::objectstore::RootEntry re (*be); - cta::objectstore::ScopedSharedLock rel(re); - re.fetch(); - cta::objectstore::AgentRegister ar(re.getAgentRegisterAddress(), *be); - rel.release(); - cta::objectstore::ScopedExclusiveLock arl(ar); - ar.fetch(); - ar.removeAgent(agentName); - ar.commit(); - std::cout << "De-listed a non-existing agent." << std::endl; - exit (EXIT_SUCCESS); - } - cta::objectstore::Agent ag(agentName, *be); - cta::objectstore::ScopedExclusiveLock agl(ag); - try { - ag.fetch(); - } catch (cta::objectstore::ObjectOpsBase::WrongType &) { - // Failure to parse an object might come for a zero size one. If this is the case, remove it - if (be->read(ag.getAddressIfSet()).empty()) { - be->remove(ag.getAddressIfSet()); - std::cout << "Agent object was empty. Removed it and exiting." << std::endl; - exit (EXIT_SUCCESS); - } - } - // Add the agent to the list of untracked agents - cta::objectstore::RootEntry re (*be); - cta::objectstore::ScopedSharedLock rel(re); - re.fetch(); - cta::objectstore::AgentRegister ar(re.getAgentRegisterAddress(), *be); - rel.release(); - // Check that the agent is indeed orphaned (owned by a non-existing owner). - // Also check that he is not an active member of a cyclic ownership. To check we follow the - // ownership chain for a limited depth. If the chain comes back to this agent, we count it - // as a active cycler. - size_t depth = 50; - std::string currentOwner = ag.getOwner(); - std::string cuttrentlyConsideredAgent = ag.getAddressIfSet(); - while (depth) { - // Simple orphan or owned by register: not a cycler. - if (ar.getAddressIfSet() == currentOwner) { - std::cout << "This agent is owner by the agent register." << std::endl; - break; - } else if (!be->exists(currentOwner)) { - std::cout << "This agent is owned by a broken chain of ownership: " << cuttrentlyConsideredAgent << "is orphaned." << std::endl; - break; - } - // Move to the next owner: - depth--; - cta::objectstore::Agent ag2(currentOwner, *be); - cta::objectstore::ScopedSharedLock ag2l(ag2); - ag2.fetch(); - cuttrentlyConsideredAgent = currentOwner; - currentOwner = ag2.getOwner(); - if (currentOwner == ag.getAddressIfSet()) { - std::cout << "This agent is a cycler." << std::endl; - goto cycler; - } - } - if ((ar.getAddressIfSet() != ag.getOwner()) && be->exists(ag.getOwner())) { - std::stringstream err; - err << "Agent not orphaned: owner object exists: " << ag.getOwner(); - throw std::runtime_error(err.str().c_str()); - } -cycler: - cta::objectstore::ScopedExclusiveLock arl(ar); - ar.fetch(); - try { - ar.untrackAgent(ag.getAddressIfSet()); - } catch (cta::objectstore::serializers::NotFound &) { - std::cout << "Agent was not known to the agent register. Re-registering it." << std::endl; - ar.addAgent(ag.getAddressIfSet()); - } - ar.commit(); - arl.release(); - ag.setOwner(ar.getAddressIfSet()); - ag.commit(); - agl.release(); - std::cout << "Agent is now listed as untracked." << std::endl; - } catch (std::exception & e) { - std::cerr << "Failed to untrack object: " - << std::endl << e.what() << std::endl; - } -} diff --git a/xroot_plugins/XrdCtaListPendingQueue.hpp b/xroot_plugins/XrdCtaListPendingQueue.hpp index 1e79244e52c25d3f3c6d181c189291dabeec3cd5..ab6a87d55bb0f57f3221c66064d3abd36b6b71c0 100644 --- a/xroot_plugins/XrdCtaListPendingQueue.hpp +++ b/xroot_plugins/XrdCtaListPendingQueue.hpp @@ -20,6 +20,7 @@ #include <xroot_plugins/XrdCtaStream.hpp> #include <xroot_plugins/XrdSsiCtaRequestMessage.hpp> +#include <common/checksum/ChecksumBlobSerDeser.hpp> namespace cta { namespace xrd { diff --git a/xroot_plugins/XrdCtaTapeFileLs.hpp b/xroot_plugins/XrdCtaTapeFileLs.hpp index 21144bc5c5140335a9dcbafbef4d408f50d1a76c..43823f0077e68ef3f3eea59af95f37ed9478b700 100644 --- a/xroot_plugins/XrdCtaTapeFileLs.hpp +++ b/xroot_plugins/XrdCtaTapeFileLs.hpp @@ -75,8 +75,35 @@ TapeFileLsStream::TapeFileLsStream(const RequestMessage &requestMsg, m_LookupNamespace = true; + bool has_any = false; // set to true if at least one optional option is set + + // Get the search criteria from the optional options cta::catalogue::TapeFileSearchCriteria searchCriteria; - searchCriteria.vid = requestMsg.getRequired(OptionString::VID); + + searchCriteria.showSuperseded = requestMsg.has_flag(OptionBoolean::SHOW_SUPERSEDED); + + searchCriteria.vid = requestMsg.getOptional(OptionString::VID, &has_any); + // Disk file IDs can be a list or a single ID + auto diskFileId = requestMsg.getOptional(OptionString::FXID, &has_any); + searchCriteria.diskFileIds = requestMsg.getOptional(OptionStrList::FILE_ID, &has_any); + if(diskFileId) { + if(!searchCriteria.diskFileIds) searchCriteria.diskFileIds = std::vector<std::string>(); + + // cta-admin converts the list from EOS fxid (hex) to fid (dec). In the case of the + // single option on the command line we need to do the conversion ourselves. + auto fid = strtol(diskFileId->c_str(), nullptr, 16); + if(fid < 1 || fid == LONG_MAX) { + throw cta::exception::UserError(*diskFileId + " is not a valid file ID"); + } + searchCriteria.diskFileIds->push_back(std::to_string(fid)); + } + // Disk instance on its own does not give a valid set of search criteria (no &has_any) + searchCriteria.diskInstance = requestMsg.getOptional(OptionString::INSTANCE); + + if(!has_any) { + throw cta::exception::UserError("Must specify at least one search option"); + } + m_tapeFileItor = m_catalogue.getArchiveFilesItor(searchCriteria); } diff --git a/xroot_plugins/XrdSsiCtaRequestMessage.cpp b/xroot_plugins/XrdSsiCtaRequestMessage.cpp index a5e6c96fcf326ee70613eff03393378ec0d646b6..6fde3828d954a323c3836eca6b52316cd0d7991f 100644 --- a/xroot_plugins/XrdSsiCtaRequestMessage.cpp +++ b/xroot_plugins/XrdSsiCtaRequestMessage.cpp @@ -23,7 +23,6 @@ using XrdSsiPb::PbException; #include <cmdline/CtaAdminCmdParse.hpp> #include "XrdSsiCtaRequestMessage.hpp" #include "XrdCtaAdminLs.hpp" -#include "XrdCtaArchiveFileLs.hpp" #include "XrdCtaArchiveRouteLs.hpp" #include "XrdCtaDriveLs.hpp" #include "XrdCtaFailedRequestLs.hpp" @@ -103,9 +102,11 @@ void RequestMessage::process(const cta::xrd::Request &request, cta::xrd::Respons case cmd_pair(AdminCmd::CMD_ADMIN, AdminCmd::SUBCMD_LS): processAdmin_Ls(response, stream); break; +#if 0 case cmd_pair(AdminCmd::CMD_ARCHIVEFILE, AdminCmd::SUBCMD_LS): processArchiveFile_Ls(response, stream); break; +#endif case cmd_pair(AdminCmd::CMD_ARCHIVEROUTE, AdminCmd::SUBCMD_ADD): processArchiveRoute_Add(response); break; @@ -848,6 +849,7 @@ void RequestMessage::processAdmin_Ls(cta::xrd::Response &response, XrdSsiStream* +#if 0 void RequestMessage::processArchiveFile_Ls(cta::xrd::Response &response, XrdSsiStream* &stream) { using namespace cta::admin; @@ -861,6 +863,7 @@ void RequestMessage::processArchiveFile_Ls(cta::xrd::Response &response, XrdSsiS response.set_type(cta::xrd::Response::RSP_SUCCESS); } +#endif diff --git a/xroot_plugins/XrdSsiCtaRequestMessage.hpp b/xroot_plugins/XrdSsiCtaRequestMessage.hpp index 4f7b6640307cf582a2d62563194053fbe7cacce1..55fa5d96352786c34b7bacaa8f6bc0f15429f7d2 100644 --- a/xroot_plugins/XrdSsiCtaRequestMessage.hpp +++ b/xroot_plugins/XrdSsiCtaRequestMessage.hpp @@ -218,7 +218,6 @@ private: typedef void admincmdstream_t(cta::xrd::Response &response, XrdSsiStream* &stream); admincmdstream_t processAdmin_Ls; - admincmdstream_t processArchiveFile_Ls; admincmdstream_t processArchiveRoute_Ls; admincmdstream_t processDrive_Ls; admincmdstream_t processFailedRequest_Ls;