diff --git a/.gitlab/issue_templates/Bug.md b/.gitlab/issue_templates/Bug.md
new file mode 100644
index 0000000000000000000000000000000000000000..14cd14aa432393ababced2f1c8597040b5f78c54
--- /dev/null
+++ b/.gitlab/issue_templates/Bug.md
@@ -0,0 +1,34 @@
+<!---
+Please read this!
+
+Before opening a new issue, make sure to search for keywords in the issues.
+
+and verify the issue you're about to submit isn't a duplicate.
+--->
+
+### Summary
+
+(Summarize the bug encountered concisely)
+
+### Steps to reproduce
+
+(How one can reproduce the issue - this is very important)
+
+### What is the current *bug* behavior?
+
+(What actually happens)
+
+### What is the expected *correct* behavior?
+
+(What you should see instead)
+
+### Relevant logs and/or screenshots
+
+(Paste any relevant logs - please use code blocks (```) to format console output,
+logs, and code as it's tough to read otherwise.)
+
+### Possible fixes
+
+(If you can, link to the line of code that might be responsible for the problem)
+
+/label ~bug
diff --git a/.gitlab/issue_templates/Feature.md b/.gitlab/issue_templates/Feature.md
new file mode 100644
index 0000000000000000000000000000000000000000..34243a26c599dcac551815c60ad98e7b3fe20888
--- /dev/null
+++ b/.gitlab/issue_templates/Feature.md
@@ -0,0 +1,17 @@
+### Problem to solve
+
+<!-- What problem do we solve? -->
+
+### Intended users
+
+<!-- Who will use this feature? If known, include any of the following: types of users (e.g. Developer, Operations), or specific company roles (e.g. Release Manager). It's okay to write "Unknown" and fill this field in later. -->
+
+### Further details
+
+<!-- Include use cases, benefits, and/or goals -->
+
+### Proposal
+
+<!-- How are we going to solve the problem? -->
+
+/label ~feature
diff --git a/catalogue/Catalogue.hpp b/catalogue/Catalogue.hpp
index 695eabebfe6c97884d521901eaa82f2ee46c7c1b..e3caf2ed1cfc999cfdf530ce55b08cdfd141006c 100644
--- a/catalogue/Catalogue.hpp
+++ b/catalogue/Catalogue.hpp
@@ -338,14 +338,6 @@ public:
* @param vid The volume identifier of the tape to be reclaimed.
*/
virtual void reclaimTape(const common::dataStructures::SecurityIdentity &admin, const std::string &vid) = 0;
-
- /**
- * This method should ONLY be used for TESTS. It does exactly the same as the real reclaimTape but it does not verify the
- * SUPERSEDED_BY_VID and SUPERSEDED_BY_FSEQ attributes
- * @param admin The administrator.
- * @param vid The volume identifier of the tape to be reclaimed.
- */
- virtual void fakeReclaimTapeForTests(const common::dataStructures::SecurityIdentity& admin, const std::string &vid) = 0;
virtual void modifyTapeMediaType(const common::dataStructures::SecurityIdentity &admin, const std::string &vid, const std::string &mediaType) = 0;
virtual void modifyTapeVendor(const common::dataStructures::SecurityIdentity &admin, const std::string &vid, const std::string &vendor) = 0;
@@ -599,16 +591,6 @@ public:
* @return True if the tape exists.
*/
virtual bool tapeExists(const std::string &vid) const = 0;
-
- /**
- * Returns true if non superseded files exist after fSeq in the tape where vid is passed in parameter
- * If there is only superseded files after fSeq, these tape files will be deleted
- *
- * @param vid the vid of the tape to check if non superseded files exist after fSeq
- * @param fSeq the fSeq after which we want to check if non superseded files exist
- * @return true if non superseded files exist, false otherwise
- */
- virtual bool existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(const std::string &vid, const uint64_t fSeq) const = 0;
}; // class Catalogue
diff --git a/catalogue/CatalogueRetryWrapper.hpp b/catalogue/CatalogueRetryWrapper.hpp
index b9155d69c4b2ddb531286f31a60550cd144fb03b..65d98eaf85da745baeeed07f636dc4d6bd5f9943 100644
--- a/catalogue/CatalogueRetryWrapper.hpp
+++ b/catalogue/CatalogueRetryWrapper.hpp
@@ -232,10 +232,6 @@ public:
void reclaimTape(const common::dataStructures::SecurityIdentity &admin, const std::string &vid) override {
return retryOnLostConnection(m_log, [&]{return m_catalogue->reclaimTape(admin, vid);}, m_maxTriesToConnect);
}
-
- void fakeReclaimTapeForTests(const common::dataStructures::SecurityIdentity &admin, const std::string &vid) override {
- return retryOnLostConnection(m_log, [&]{return m_catalogue->fakeReclaimTapeForTests(admin, vid);}, m_maxTriesToConnect);
- }
void modifyTapeMediaType(const common::dataStructures::SecurityIdentity &admin, const std::string &vid, const std::string &mediaType) override {
return retryOnLostConnection(m_log, [&]{return m_catalogue->modifyTapeMediaType(admin, vid, mediaType);}, m_maxTriesToConnect);
@@ -388,11 +384,6 @@ public:
bool tapeExists(const std::string &vid) const override {
return retryOnLostConnection(m_log, [&]{return m_catalogue->tapeExists(vid);}, m_maxTriesToConnect);
}
-
- bool existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(const std::string& vid, const uint64_t fSeq) const override {
- return retryOnLostConnection(m_log,[&]{return m_catalogue->existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(vid,fSeq);},m_maxTriesToConnect);
- }
-
protected:
diff --git a/catalogue/CatalogueTest.cpp b/catalogue/CatalogueTest.cpp
index 97000ca464a268a50cb3f91811cc32ff7de48fae..7e98967beab39de7ede5f95c38fcf0d5b27c7309 100644
--- a/catalogue/CatalogueTest.cpp
+++ b/catalogue/CatalogueTest.cpp
@@ -12151,122 +12151,6 @@ TEST_P(cta_catalogue_CatalogueTest, reclaimTape_full_lastFSeq_1_one_tape_file_su
}
}
-TEST_P(cta_catalogue_CatalogueTest, exist_non_superseded_files_after_fseq) {
- using namespace cta;
-
- const std::string diskInstanceName1 = "disk_instance_1";
-
- ASSERT_TRUE(m_catalogue->getTapes().empty());
-
- const std::string vid1 = "VID123";
- const std::string vid2 = "VID234";
- const std::string mediaType = "media_type";
- const std::string vendor = "vendor";
- const std::string logicalLibraryName = "logical_library_name";
- const bool logicalLibraryIsDisabled= false;
- const std::string tapePoolName = "tape_pool_name";
- const std::string vo = "vo";
- const uint64_t nbPartialTapes = 2;
- const bool isEncrypted = true;
- const cta::optional<std::string> supply("value for the supply pool mechanism");
- const uint64_t capacityInBytes = (uint64_t)10 * 1000 * 1000 * 1000 * 1000;
- const bool disabledValue = true;
- const bool fullValue = false;
- const std::string createTapeComment = "Create tape";
-
- m_catalogue->createLogicalLibrary(m_admin, logicalLibraryName, logicalLibraryIsDisabled, "Create logical library");
- m_catalogue->createTapePool(m_admin, tapePoolName, vo, nbPartialTapes, isEncrypted, supply, "Create tape pool");
- m_catalogue->createTape(m_admin, vid1, mediaType, vendor, logicalLibraryName, tapePoolName, capacityInBytes,
- disabledValue, fullValue, createTapeComment);
-
- //A tape with no tape file have no files after FSeq 0
- ASSERT_FALSE(m_catalogue->existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(vid1,0));
-
- const uint64_t archiveFileId = 1234;
-
- ASSERT_FALSE(m_catalogue->getArchiveFilesItor().hasMore());
- ASSERT_THROW(m_catalogue->getArchiveFileById(archiveFileId), exception::Exception);
-
- common::dataStructures::StorageClass storageClass;
- storageClass.diskInstance = diskInstanceName1;
- storageClass.name = "storage_class";
- storageClass.nbCopies = 1;
- storageClass.comment = "Create storage class";
- m_catalogue->createStorageClass(m_admin, storageClass);
-
- /*
- * Insert a file in the tape vid1
- */
- {
- const uint64_t archiveFileSize = 1;
- const std::string tapeDrive = "tape_drive";
- const std::string checksumType = "checksum_type";
- const std::string checksumValue = "checksum_value";
-
- auto file1WrittenUP=cta::make_unique<cta::catalogue::TapeFileWritten>();
- auto & file1Written = *file1WrittenUP;
- std::set<cta::catalogue::TapeItemWrittenPointer> file1WrittenSet;
- file1WrittenSet.insert(file1WrittenUP.release());
- file1Written.archiveFileId = archiveFileId;
- file1Written.diskInstance = storageClass.diskInstance;
- file1Written.diskFileId = "5678";
- file1Written.diskFilePath = "/public_dir/public_file";
- file1Written.diskFileUser = "public_disk_user";
- file1Written.diskFileGroup = "public_disk_group";
- file1Written.size = archiveFileSize;
- file1Written.checksumType = checksumType;
- file1Written.checksumValue = checksumValue;
- file1Written.storageClassName = storageClass.name;
- file1Written.vid = vid1;
- file1Written.fSeq = 1;
- file1Written.blockId = 4321;
- file1Written.compressedSize = 1;
- file1Written.copyNb = 1;
- file1Written.tapeDrive = tapeDrive;
- m_catalogue->filesWrittenToTape(file1WrittenSet);
- }
- //One file written : this file is not superseded by another one, existNonSupersededFilesAfterFSeq = true
- ASSERT_TRUE(m_catalogue->existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(vid1,0));
- //No file after the only file inserted, existNonSupersededFilesAfterFseq = false
- ASSERT_FALSE(m_catalogue->existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(vid1,1));
-
- //Insert another file in another tape that will supersed the first one in vid1
- {
- m_catalogue->createTape(m_admin, vid2, mediaType, vendor, logicalLibraryName, tapePoolName, capacityInBytes,
- disabledValue, fullValue, createTapeComment);
- const uint64_t archiveFileSize = 1;
- const std::string tapeDrive = "tape_drive";
- const std::string checksumType = "checksum_type";
- const std::string checksumValue = "checksum_value";
-
- auto file1WrittenUP=cta::make_unique<cta::catalogue::TapeFileWritten>();
- auto & file1Written = *file1WrittenUP;
- std::set<cta::catalogue::TapeItemWrittenPointer> file1WrittenSet;
- file1WrittenSet.insert(file1WrittenUP.release());
- file1Written.archiveFileId = archiveFileId;
- file1Written.diskInstance = storageClass.diskInstance;
- file1Written.diskFileId = "5678";
- file1Written.diskFilePath = "/public_dir/public_file";
- file1Written.diskFileUser = "public_disk_user";
- file1Written.diskFileGroup = "public_disk_group";
- file1Written.size = archiveFileSize;
- file1Written.checksumType = checksumType;
- file1Written.checksumValue = checksumValue;
- file1Written.storageClassName = storageClass.name;
- file1Written.vid = vid2;
- file1Written.fSeq = 1;
- file1Written.blockId = 4321;
- file1Written.compressedSize = 1;
- file1Written.copyNb = 1;
- file1Written.tapeDrive = tapeDrive;
- m_catalogue->filesWrittenToTape(file1WrittenSet);
- }
- //The tape files written to tape vid2 are not superseded by any file, but the tape files in vid1
- //are superseded by the tape files in vid2
- ASSERT_FALSE(m_catalogue->existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(vid1,0));
- ASSERT_TRUE(m_catalogue->existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(vid2,0));
-}
-
TEST_P(cta_catalogue_CatalogueTest, ping) {
using namespace cta;
diff --git a/catalogue/DummyCatalogue.hpp b/catalogue/DummyCatalogue.hpp
index a67b479e1710340e5726459b69beefbe4aebda3f..5e83c60901025a45c9803006f015e233617a0311 100644
--- a/catalogue/DummyCatalogue.hpp
+++ b/catalogue/DummyCatalogue.hpp
@@ -107,7 +107,6 @@ public:
const std::string &storageClassName, const common::dataStructures::UserIdentity &user) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
common::dataStructures::RetrieveFileQueueCriteria prepareToRetrieveFile(const std::string& instanceName, const uint64_t archiveFileId, const common::dataStructures::UserIdentity& user, log::LogContext &lc) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
void reclaimTape(const common::dataStructures::SecurityIdentity& admin, const std::string& vid) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
- void fakeReclaimTapeForTests(const common::dataStructures::SecurityIdentity& admin, const std::string& vid) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
void setTapeDisabled(const common::dataStructures::SecurityIdentity& admin, const std::string& vid, const bool disabledValue) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
void setTapeFull(const common::dataStructures::SecurityIdentity& admin, const std::string& vid, const bool fullValue) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
void setTapePoolEncryption(const common::dataStructures::SecurityIdentity& admin, const std::string& name, const bool encryptionValue) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
@@ -116,7 +115,6 @@ public:
void tapeMountedForArchive(const std::string& vid, const std::string& drive) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
void tapeMountedForRetrieve(const std::string& vid, const std::string& drive) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
bool tapePoolExists(const std::string& tapePoolName) const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
- bool existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(const std::string& vid, const uint64_t fSeq) const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
// Special functions for unit tests.
void addEnabledTape(const std::string & vid) {
diff --git a/catalogue/RdbmsCatalogue.cpp b/catalogue/RdbmsCatalogue.cpp
index 4ad7b64be62195aac45f256293e23f88be420e48..fc5424be8cfd910d457a65f3fc8d045932936a58 100644
--- a/catalogue/RdbmsCatalogue.cpp
+++ b/catalogue/RdbmsCatalogue.cpp
@@ -1891,47 +1891,6 @@ bool RdbmsCatalogue::tapeExists(rdbms::Conn &conn, const std::string &vid) const
}
}
-bool RdbmsCatalogue::existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(const std::string& vid, const uint64_t fSeq) const {
- try{
- auto conn = m_connPool.getConn();
- const char *const sql =
- "SELECT VID "
- "FROM TAPE_FILE "
- "WHERE "
- "VID = :VID AND "
- "FSEQ > :FSEQ AND "
- "SUPERSEDED_BY_VID IS NULL AND "
- "SUPERSEDED_BY_FSEQ IS NULL";
- auto stmt = conn.createStmt(sql);
- stmt.bindString(":VID",vid);
- stmt.bindUint64(":FSEQ",fSeq);
- auto rset = stmt.executeQuery();
- if(!rset.next()){
- //No non-superseded files detected, we can delete the tape files
- const char *const sqlDelete =
- "DELETE FROM TAPE_FILE "
- "WHERE "
- "VID =:VID AND "
- "FSEQ > :FSEQ AND "
- "SUPERSEDED_BY_VID IS NOT NULL AND "
- "SUPERSEDED_BY_FSEQ IS NOT NULL";
- auto stmtDelete = conn.createStmt(sqlDelete);
- stmtDelete.bindString(":VID",vid);
- stmtDelete.bindUint64(":FSEQ",fSeq);
- stmtDelete.executeNonQuery();
- return false;
- }
- //Non superseded files are present
- return true;
- } catch(exception::UserError &) {
- throw;
- } catch(exception::Exception &ex) {
- ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str());
- throw;
- }
-}
-
-
//------------------------------------------------------------------------------
// deleteTape
//------------------------------------------------------------------------------
@@ -2346,73 +2305,47 @@ common::dataStructures::VidToTapeMap RdbmsCatalogue::getAllTapes() const {
}
//------------------------------------------------------------------------------
-// reclaimTape
+//getNbNonSupersededFilesOnTape
//------------------------------------------------------------------------------
-void RdbmsCatalogue::reclaimTape(const common::dataStructures::SecurityIdentity &admin, const std::string &vid) {
+uint64_t RdbmsCatalogue::getNbNonSupersededFilesOnTape(rdbms::Conn& conn, const std::string& vid) const {
try {
- const time_t now = time(nullptr);
- const char *const sql =
- "UPDATE TAPE SET "
- "DATA_IN_BYTES = 0,"
- "LAST_FSEQ = 0,"
- "IS_FULL = '0',"
- "LAST_UPDATE_USER_NAME = :LAST_UPDATE_USER_NAME,"
- "LAST_UPDATE_HOST_NAME = :LAST_UPDATE_HOST_NAME,"
- "LAST_UPDATE_TIME = :LAST_UPDATE_TIME "
- "WHERE "
- "VID = :UPDATE_VID AND "
- "IS_FULL != '0' AND "
- "NOT EXISTS (SELECT VID FROM TAPE_FILE WHERE VID = :SELECT_VID "
- " AND SUPERSEDED_BY_VID IS NULL "
- " AND SUPERSEDED_BY_FSEQ IS NULL)";
- auto conn = m_connPool.getConn();
+ const char *const sql =
+ "SELECT COUNT(*) AS NB_NON_SUPERSEDED_FILES FROM TAPE_FILE "
+ "WHERE VID = :VID "
+ "AND SUPERSEDED_BY_VID IS NULL "
+ "AND SUPERSEDED_BY_FSEQ IS NULL";
auto stmt = conn.createStmt(sql);
- stmt.bindString(":LAST_UPDATE_USER_NAME", admin.username);
- stmt.bindString(":LAST_UPDATE_HOST_NAME", admin.host);
- stmt.bindUint64(":LAST_UPDATE_TIME", now);
- stmt.bindString(":UPDATE_VID", vid);
- stmt.bindString(":SELECT_VID", vid);
- stmt.executeNonQuery();
-
- // If the update failed due to a user error
- if(0 == stmt.getNbAffectedRows()) {
- // Try to determine the user error
- //
- // Please note that this is a best effort diagnosis because there is no
- // lock on the database to prevent other concurrent updates from taking
- // place on the TAPE and TAPE_FILE tables
- TapeSearchCriteria searchCriteria;
- searchCriteria.vid = vid;
- const auto tapes = getTapes(conn, searchCriteria);
-
- if(tapes.empty()) {
- throw exception::UserError(std::string("Cannot reclaim tape ") + vid + " because it does not exist");
- } else {
- if(!tapes.front().full) {
- throw exception::UserError(std::string("Cannot reclaim tape ") + vid + " because it is not FULL");
- } else {
- throw exception::UserError(std::string("Cannot reclaim tape ") + vid + " because there is at least one tape"
- " file in the catalogue that is on the tape");
- }
- }
- }
- } catch(exception::UserError &) {
- throw;
+ stmt.bindString(":VID", vid);
+ auto rset = stmt.executeQuery();
+ rset.next();
+ return rset.columnUint64("NB_NON_SUPERSEDED_FILES");
} catch(exception::Exception &ex) {
ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str());
throw;
}
}
-//------------------------------------------------------------------------------
-//fakeReclaimTapeForTests
-//------------------------------------------------------------------------------
-void RdbmsCatalogue::fakeReclaimTapeForTests(const common::dataStructures::SecurityIdentity& admin, const std::string& vid) {
+void RdbmsCatalogue::deleteTapeFiles(rdbms::Conn& conn, const std::string& vid) const {
+ try {
+ const char * const sql =
+ "DELETE FROM TAPE_FILE WHERE VID = :VID "
+ "AND SUPERSEDED_BY_VID IS NOT NULL "
+ "AND SUPERSEDED_BY_FSEQ IS NOT NULL";
+ auto stmt = conn.createStmt(sql);
+ stmt.bindString(":VID", vid);
+ stmt.executeNonQuery();
+ } catch(exception::Exception &ex) {
+ ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str());
+ throw;
+ }
+}
+
+void RdbmsCatalogue::resetTapeCounters(rdbms::Conn& conn, const common::dataStructures::SecurityIdentity& admin, const std::string& vid) const {
try {
const time_t now = time(nullptr);
- const char *const sql =
- "UPDATE TAPE SET "
+ const char * const sql =
+ "UPDATE TAPE SET "
"DATA_IN_BYTES = 0,"
"LAST_FSEQ = 0,"
"IS_FULL = '0',"
@@ -2420,47 +2353,55 @@ void RdbmsCatalogue::fakeReclaimTapeForTests(const common::dataStructures::Secur
"LAST_UPDATE_HOST_NAME = :LAST_UPDATE_HOST_NAME,"
"LAST_UPDATE_TIME = :LAST_UPDATE_TIME "
"WHERE "
- "VID = :UPDATE_VID AND "
- "IS_FULL != '0'";
- auto conn = m_connPool.getConn();
+ "VID = :VID";
auto stmt = conn.createStmt(sql);
stmt.bindString(":LAST_UPDATE_USER_NAME", admin.username);
stmt.bindString(":LAST_UPDATE_HOST_NAME", admin.host);
stmt.bindUint64(":LAST_UPDATE_TIME", now);
- stmt.bindString(":UPDATE_VID", vid);
+ stmt.bindString(":VID", vid);
stmt.executeNonQuery();
+ } catch (exception::Exception &ex){
+ ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str());
+ throw;
+ }
+}
- // If the update failed due to a user error
- if(0 == stmt.getNbAffectedRows()) {
- // Try to determine the user error
- //
- // Please note that this is a best effort diagnosis because there is no
- // lock on the database to prevent other concurrent updates from taking
- // place on the TAPE and TAPE_FILE tables
- TapeSearchCriteria searchCriteria;
- searchCriteria.vid = vid;
- const auto tapes = getTapes(conn, searchCriteria);
-
- if(tapes.empty()) {
- throw exception::UserError(std::string("Cannot reclaim tape ") + vid + " because it does not exist");
- } else {
- if(!tapes.front().full) {
- throw exception::UserError(std::string("Cannot reclaim tape ") + vid + " because it is not FULL");
- } else {
- throw exception::UserError(std::string("Cannot reclaim tape ") + vid + " because there is at least one tape"
+//------------------------------------------------------------------------------
+// reclaimTape
+//------------------------------------------------------------------------------
+void RdbmsCatalogue::reclaimTape(const common::dataStructures::SecurityIdentity &admin, const std::string &vid) {
+ try{
+ auto conn = m_connPool.getConn();
+
+ TapeSearchCriteria searchCriteria;
+ searchCriteria.vid = vid;
+ const auto tapes = getTapes(conn, searchCriteria);
+
+ if(tapes.empty()) {
+ throw exception::UserError(std::string("Cannot reclaim tape ") + vid + " because it does not exist");
+ } else {
+ if(!tapes.front().full){
+ throw exception::UserError(std::string("Cannot reclaim tape ") + vid + " because it is not FULL");
+ }
+ }
+ //The tape exists and is full, we can try to reclaim it
+ if(getNbNonSupersededFilesOnTape(conn,vid) == 0){
+ //There is no non-superseded files on the tape, we can reclaim it : delete the files and reset the counters
+ deleteTapeFiles(conn,vid);
+ resetTapeCounters(conn,admin,vid);
+ } else {
+ throw exception::UserError(std::string("Cannot reclaim tape ") + vid + " because there is at least one tape"
" file in the catalogue that is on the tape");
- }
- }
}
- } catch(exception::UserError &) {
+ } catch (exception::UserError& ue) {
throw;
- } catch(exception::Exception &ex) {
+ }
+ catch (exception::Exception &ex) {
ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str());
throw;
}
}
-
//------------------------------------------------------------------------------
// getTapeLogFromRset
//------------------------------------------------------------------------------
diff --git a/catalogue/RdbmsCatalogue.hpp b/catalogue/RdbmsCatalogue.hpp
index 766dca01f7390bd4176adcd2e7535998b8a3431b..584074885cb595fa34ef76c0b16ac1de35f02e9a 100644
--- a/catalogue/RdbmsCatalogue.hpp
+++ b/catalogue/RdbmsCatalogue.hpp
@@ -332,7 +332,27 @@ public:
* @param vid The volume identifier of the tape to be reclaimed.
*/
void reclaimTape(const common::dataStructures::SecurityIdentity &admin, const std::string &vid) override;
- void fakeReclaimTapeForTests(const common::dataStructures::SecurityIdentity& admin, const std::string& vid) override;
+ /**
+ * Returns the number of non superseded files contained in the tape identified by its vid
+ * @param conn the database connection
+ * @param vid the vid in which we will count non superseded files
+ * @return the number of non superseded files on the vid
+ */
+ uint64_t getNbNonSupersededFilesOnTape(rdbms::Conn &conn, const std::string &vid) const;
+ /**
+ * Delete all the tape files of the VID passed in parameter
+ * @param conn the database connection
+ * @param vid the vid in which we want to remove all the tape files
+ */
+ void deleteTapeFiles(rdbms::Conn &conn, const std::string& vid) const;
+
+ /**
+ * Reset the counters of a tape
+ * @param conn the database connection
+ * @param admin the administrator
+ * @param vid the vid to reset the counters
+ */
+ void resetTapeCounters(rdbms::Conn &conn, const common::dataStructures::SecurityIdentity &admin ,const std::string& vid) const;
void modifyTapeMediaType(const common::dataStructures::SecurityIdentity &admin, const std::string &vid, const std::string &mediaType) override;
void modifyTapeVendor(const common::dataStructures::SecurityIdentity &admin, const std::string &vid, const std::string &vendor) override;
void modifyTapeLogicalLibraryName(const common::dataStructures::SecurityIdentity &admin, const std::string &vid, const std::string &logicalLibraryName) override;
@@ -715,17 +735,6 @@ protected:
* @return True if the tape exists.
*/
bool tapeExists(rdbms::Conn &conn, const std::string &vid) const;
-
- /**
- * Returns true if non superseded files exist after fSeq in the tape where vid is passed in parameter
- * If there is only superseded files after fSeq, these tape files will be deleted
- *
- * @param vid the vid of the tape to check if non superseded files exist after fSeq
- * @param fSeq the fSeq after which we want to check if non superseded files exist
- * @return true if non superseded files exist, false otherwise
- */
- bool existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(const std::string& vid, const uint64_t fSeq) const override;
-
/**
* Returns the list of tapes that meet the specified search criteria.
@@ -735,7 +744,7 @@ protected:
* @return The list of tapes.
*/
std::list<common::dataStructures::Tape> getTapes(rdbms::Conn &conn, const TapeSearchCriteria &searchCriteria) const;
-
+
/**
* Returns true if the specified logical library exists.
*
diff --git a/catalogue/common_catalogue_schema.sql b/catalogue/common_catalogue_schema.sql
index e37a2d5022b9a3b87d071002acff2f1171485b29..a5f2a3b80170a6d71f2d5ad4de64a8fe835c2d84 100644
--- a/catalogue/common_catalogue_schema.sql
+++ b/catalogue/common_catalogue_schema.sql
@@ -75,17 +75,17 @@ CREATE TABLE LOGICAL_LIBRARY(
CONSTRAINT LOGICAL_LIBRARY_ID_BOOL_CK CHECK(IS_DISABLED IN ('0', '1'))
);
CREATE TABLE TAPE(
- VID VARCHAR(100) CONSTRAINT TAPE_V_UN NOT NULL,
+ VID VARCHAR(100) CONSTRAINT TAPE_V_NN NOT NULL,
MEDIA_TYPE VARCHAR(100) CONSTRAINT TAPE_MT_NN NOT NULL,
VENDOR VARCHAR(100) CONSTRAINT TAPE_V2_NN NOT NULL,
- LOGICAL_LIBRARY_NAME VARCHAR(100) CONSTRAINT TAPE_LLN_UN NOT NULL,
- TAPE_POOL_NAME VARCHAR(100) CONSTRAINT TAPE_TPN_UN NOT NULL,
+ LOGICAL_LIBRARY_NAME VARCHAR(100) CONSTRAINT TAPE_LLN_NN NOT NULL,
+ TAPE_POOL_NAME VARCHAR(100) CONSTRAINT TAPE_TPN_NN NOT NULL,
ENCRYPTION_KEY VARCHAR(100),
- CAPACITY_IN_BYTES NUMERIC(20, 0) CONSTRAINT TAPE_CIB_UN NOT NULL,
- DATA_IN_BYTES NUMERIC(20, 0) CONSTRAINT TAPE_DIB_UN NOT NULL,
- LAST_FSEQ NUMERIC(20, 0) CONSTRAINT TAPE_LF_UN NOT NULL,
- IS_DISABLED CHAR(1) CONSTRAINT TAPE_ID_UN NOT NULL,
- IS_FULL CHAR(1) CONSTRAINT TAPE_IF_UN NOT NULL,
+ CAPACITY_IN_BYTES NUMERIC(20, 0) CONSTRAINT TAPE_CIB_NN NOT NULL,
+ DATA_IN_BYTES NUMERIC(20, 0) CONSTRAINT TAPE_DIB_NN NOT NULL,
+ LAST_FSEQ NUMERIC(20, 0) CONSTRAINT TAPE_LF_NN NOT NULL,
+ IS_DISABLED CHAR(1) CONSTRAINT TAPE_ID_NN NOT NULL,
+ IS_FULL CHAR(1) CONSTRAINT TAPE_IF_NN NOT NULL,
LABEL_DRIVE VARCHAR(100),
LABEL_TIME NUMERIC(20, 0),
LAST_READ_DRIVE VARCHAR(100),
@@ -94,13 +94,13 @@ CREATE TABLE TAPE(
LAST_WRITE_TIME NUMERIC(20, 0),
READ_MOUNT_COUNT NUMERIC(20, 0) DEFAULT 0 CONSTRAINT TAPE_RMC_NN NOT NULL,
WRITE_MOUNT_COUNT NUMERIC(20, 0) DEFAULT 0 CONSTRAINT TAPE_WMC_NN NOT NULL,
- USER_COMMENT VARCHAR(1000) CONSTRAINT TAPE_UC_UN NOT NULL,
- CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT TAPE_CLUN_UN NOT NULL,
- CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT TAPE_CLHN_UN NOT NULL,
+ USER_COMMENT VARCHAR(1000) CONSTRAINT TAPE_UC_NN NOT NULL,
+ CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT TAPE_CLUN_NN NOT NULL,
+ CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT TAPE_CLHN_NN NOT NULL,
CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT TAPE_CLT_NN NOT NULL,
- LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT TAPE_LUUN_UN NOT NULL,
- LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT TAPE_LUHN_UN NOT NULL,
- LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT TAPE_LUT_UN NOT NULL,
+ LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT TAPE_LUUN_NN NOT NULL,
+ LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT TAPE_LUHN_NN NOT NULL,
+ LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT TAPE_LUT_NN NOT NULL,
CONSTRAINT TAPE_PK PRIMARY KEY(VID),
CONSTRAINT TAPE_LOGICAL_LIBRARY_FK FOREIGN KEY(LOGICAL_LIBRARY_NAME)
REFERENCES LOGICAL_LIBRARY(LOGICAL_LIBRARY_NAME),
diff --git a/cmdline/CtaAdminCmd.cpp b/cmdline/CtaAdminCmd.cpp
index 338e2a25083fd2df654a64e461716522ecd139c6..47fe7a749a0a367d4c9dec82793cf36d5168dae9 100644
--- a/cmdline/CtaAdminCmd.cpp
+++ b/cmdline/CtaAdminCmd.cpp
@@ -78,6 +78,7 @@ void IStreamBuffer<cta::xrd::Data>::DataCallback(cta::xrd::Data record) const
case Data::kLprSummary: std::cout << Log::DumpProtobuf(&record.lpr_summary()); break;
case Data::kTplsItem: std::cout << Log::DumpProtobuf(&record.tpls_item()); break;
case Data::kTalsItem: std::cout << Log::DumpProtobuf(&record.tals_item()); break;
+ case Data::kRelsItem: std::cout << Log::DumpProtobuf(&record.rels_item()); break;
default:
throw std::runtime_error("Received invalid stream data from CTA Frontend.");
}
@@ -94,6 +95,7 @@ void IStreamBuffer<cta::xrd::Data>::DataCallback(cta::xrd::Data record) const
case Data::kLprSummary: CtaAdminCmd::print(record.lpr_summary()); break;
case Data::kTplsItem: CtaAdminCmd::print(record.tpls_item()); break;
case Data::kTalsItem: CtaAdminCmd::print(record.tals_item()); break;
+ case Data::kRelsItem: CtaAdminCmd::print(record.rels_item()); break;
default:
throw std::runtime_error("Received invalid stream data from CTA Frontend.");
}
@@ -237,6 +239,7 @@ void CtaAdminCmd::send() const
case HeaderType::LISTPENDINGRETRIEVES_SUMMARY: printLprSummaryHeader(); break;
case HeaderType::TAPEPOOL_LS: printTpLsHeader(); break;
case HeaderType::TAPE_LS: printTapeLsHeader(); break;
+ case HeaderType::REPACK_LS: printRepackLsHeader(); break;
case HeaderType::NONE:
default: break;
}
@@ -731,6 +734,44 @@ void CtaAdminCmd::print(const cta::admin::TapeLsItem &tals_item){
<< std::endl;
}
+void CtaAdminCmd::printRepackLsHeader(){
+ std::cout << TEXT_RED
+ << std::setfill(' ') << std::setw(7) << std::right << "vid" << ' '
+ << std::setfill(' ') << std::setw(50) << std::right << "repackBufferURL" << ' '
+ << std::setfill(' ') << std::setw(17) << std::right << "userProvidedFiles" << ' '
+ << std::setfill(' ') << std::setw(20) << std::right << "totalFilesToRetrieve" << ' '
+ << std::setfill(' ') << std::setw(19) << std::right << "totalBytesToRetrieve" << ' '
+ << std::setfill(' ') << std::setw(20) << std::right << "totalFilesToArchive" << ' '
+ << std::setfill(' ') << std::setw(19) << std::right << "totalBytesToArchive" << ' '
+ << std::setfill(' ') << std::setw(14) << std::right << "retrievedFiles" << ' '
+ << std::setfill(' ') << std::setw(13) << std::right << "archivedFiles" << ' '
+ << std::setfill(' ') << std::setw(21) << std::right << "failedToRetrieveFiles" << ' '
+ << std::setfill(' ') << std::setw(20) << std::right << "failedToRetrieveBytes" << ' '
+ << std::setfill(' ') << std::setw(20) << std::right << "failedToArchiveFiles" << ' '
+ << std::setfill(' ') << std::setw(20) << std::right << "failedToArchiveBytes" << ' '
+ << std::setfill(' ') << std::setw(16) << std::right << "lastExpandedFSeq" << ' '
+ << "status" << ' '
+ << TEXT_NORMAL << std::endl;
+}
+
+void CtaAdminCmd::print(const cta::admin::RepackLsItem &rels_item){
+ std::cout << std::setfill(' ') << std::setw(7) << std::right << rels_item.vid() << ' '
+ << std::setfill(' ') << std::setw(50) << std::right << rels_item.repack_buffer_url() << ' '
+ << std::setfill(' ') << std::setw(17) << std::right << rels_item.user_provided_files() << ' '
+ << std::setfill(' ') << std::setw(20) << std::right << rels_item.total_files_to_retrieve() << ' '
+ << std::setfill(' ') << std::setw(19) << std::right << rels_item.total_bytes_to_retrieve() << ' '
+ << std::setfill(' ') << std::setw(20) << std::right << rels_item.total_files_to_archive() << ' '
+ << std::setfill(' ') << std::setw(19) << std::right << rels_item.total_bytes_to_archive() << ' '
+ << std::setfill(' ') << std::setw(14) << std::right << rels_item.retrieved_files() << ' '
+ << std::setfill(' ') << std::setw(13) << std::right << rels_item.archived_files() << ' '
+ << std::setfill(' ') << std::setw(21) << std::right << rels_item.failed_to_retrieve_files() << ' '
+ << std::setfill(' ') << std::setw(20) << std::right << rels_item.failed_to_retrieve_bytes() << ' '
+ << std::setfill(' ') << std::setw(20) << std::right << rels_item.failed_to_archive_files() << ' '
+ << std::setfill(' ') << std::setw(20) << std::right << rels_item.failed_to_retrieve_bytes() << ' '
+ << std::setfill(' ') << std::setw(10) << std::right << rels_item.last_expanded_fseq() << ' '
+ << rels_item.status() << std::endl;
+}
+
void CtaAdminCmd::print(const cta::admin::TapePoolLsItem &tpls_item)
{
std::string encrypt_str = tpls_item.encrypt() ? "true" : "false";
diff --git a/cmdline/CtaAdminCmd.hpp b/cmdline/CtaAdminCmd.hpp
index 68e1f285c0ba2c8fa306045623789f2c2e25a670..56cbc901b6c2058c9328c255cd568661e1cb86a3 100644
--- a/cmdline/CtaAdminCmd.hpp
+++ b/cmdline/CtaAdminCmd.hpp
@@ -58,6 +58,7 @@ public:
static void printLprSummaryHeader();
static void printTpLsHeader();
static void printTapeLsHeader();
+ static void printRepackLsHeader();
// Output records
static void print(const ArchiveFileLsItem &afls_item);
@@ -70,6 +71,7 @@ public:
static void print(const ListPendingRetrievesSummary &lpr_summary);
static void print(const TapePoolLsItem &tpls_item);
static void print(const cta::admin::TapeLsItem &tals_item);
+ static void print(const cta::admin::RepackLsItem &rels_item);
private:
//! Parse the options for a specific command/subcommand
diff --git a/common/dataStructures/RepackInfo.hpp b/common/dataStructures/RepackInfo.hpp
index e94e87246db7c47cecc94bb74aaddb237567561b..03b7666a0121a31a4d95419f2bbdcbdcd0ea18a4 100644
--- a/common/dataStructures/RepackInfo.hpp
+++ b/common/dataStructures/RepackInfo.hpp
@@ -58,6 +58,8 @@ struct RepackInfo {
uint64_t failedBytesToRetrieve;
uint64_t lastExpandedFseq;
uint64_t userProvidedFiles;
+ uint64_t retrievedFiles;
+ uint64_t archivedFiles;
bool isExpandFinished;
// std::string tag;
// uint64_t totalFiles;
diff --git a/common/log/Logger.cpp b/common/log/Logger.cpp
index 37638dec6567c53b7f4c53e1ffd594f6f773eadf..a3fb0e586f25af3eb637bfec5db9d6d451aee8b5 100644
--- a/common/log/Logger.cpp
+++ b/common/log/Logger.cpp
@@ -120,14 +120,14 @@ std::map<int, std::string>
std::map<int, std::string> m;
try {
- m[LOG_EMERG] = "Emerg";
- m[ALERT] = "Alert";
- m[LOG_CRIT] = "Crit";
- m[ERR] = "Error";
- m[WARNING] = "Warn";
- m[LOG_NOTICE] = "Notice";
- m[INFO] = "Info";
- m[DEBUG] = "Debug";
+ m[LOG_EMERG] = "EMERG";
+ m[ALERT] = "ALERT";
+ m[LOG_CRIT] = "CRIT";
+ m[ERR] = "ERROR";
+ m[WARNING] = "WARN";
+ m[LOG_NOTICE] = "NOTICE";
+ m[INFO] = "INFO";
+ m[DEBUG] = "DEBUG";
} catch(std::exception &se) {
exception::Exception ex;
ex.getMessage() << "Failed to generate priority to text mapping: " <<
diff --git a/continuousintegration/orchestration/tests/repack.sh b/continuousintegration/orchestration/tests/repack.sh
deleted file mode 100755
index 6336b64e8d9e470cd05e5ea6ecfa9c657c994b09..0000000000000000000000000000000000000000
--- a/continuousintegration/orchestration/tests/repack.sh
+++ /dev/null
@@ -1,68 +0,0 @@
-#!/bin/bash
-
-usage() { cat <<EOF 1>&2
-Usage: $0 -n <namespace>
-EOF
-exit 1
-}
-
-while getopts "n:" o; do
- case "${o}" in
- n)
- NAMESPACE=${OPTARG}
- ;;
- *)
- usage
- ;;
- esac
-done
-shift $((OPTIND-1))
-
-if [ -z "${NAMESPACE}" ]; then
- usage
-fi
-
-if [ ! -z "${error}" ]; then
- echo -e "ERROR:\n${error}"
- exit 1
-fi
-
-echo "Preparing namespace for the tests"
-./prepare_tests.sh -n ${NAMESPACE}
-
-kubectl -n ${NAMESPACE} cp client_helper.sh client:/root/client_helper.sh
-
-NB_FILES=1000
-FILE_SIZE_KB=15
-
-echo
-echo "Launching client_ar.sh on client pod"
-echo " Archiving ${NB_FILES} files of ${FILE_SIZE_KB}kB each"
-echo " Archiving files: xrdcp as user1"
-kubectl -n ${NAMESPACE} cp client_ar.sh client:/root/client_ar.sh
-kubectl -n ${NAMESPACE} exec client -- bash /root/client_ar.sh -n ${NB_FILES} -s ${FILE_SIZE_KB} -p 100 -d /eos/ctaeos/preprod -v -A || exit 1
-
-#kubectl -n ${NAMESPACE} exec ctaeos -- bash /root/grep_xrdlog_mgm_for_error.sh || exit 1
-
-source ./repack_helper.sh
-
-vidToRepack1=$(getFirstVidContainingFiles)
-if [ "$vidToRepack1" != "null" ]
-then
- echo
- echo "Launching a repack on tape $vidToRepack1"
- echo
- writeTapeSummary $vidToRepack1
- executeRepack $vidToRepack1
- echo
- echo "Reclaiming tape $vidToRepack1"
- executeReclaim $vidToRepack1
- echo
- writeTapeSummary $vidToRepack1
-else
- echo "No vid found to repack"
- exit 1
-fi
-
-vidDestination=$(getFirstVidContainingFiles)
-writeTapeSummary $vidDestination
\ No newline at end of file
diff --git a/continuousintegration/orchestration/tests/repack_generate_report.sh b/continuousintegration/orchestration/tests/repack_generate_report.sh
new file mode 100755
index 0000000000000000000000000000000000000000..8182ca5c79ec8233d1cf0cacb832149d8c1eaa99
--- /dev/null
+++ b/continuousintegration/orchestration/tests/repack_generate_report.sh
@@ -0,0 +1,97 @@
+#!/bin/bash
+
+REPORT_DIRECTORY=/var/log
+
+die() {
+ echo "$@" 1>&2
+ test -z $TAILPID || kill ${TAILPID} &> /dev/null
+ exit 1
+}
+
+usage() { cat <<EOF 1>&2
+Usage: $0 -v <vid> [-r <report_directory>]
+Default report_directory = ${REPORT_DIRECTORY}
+EOF
+exit 1
+}
+
+if [ $# -lt 1 ]
+then
+ usage
+fi;
+
+while getopts "v:r:" o; do
+ case "${o}" in
+ v)
+ VID=${OPTARG}
+ ;;
+ r)
+ REPORT_DIRECTORY=${OPTARG}
+ ;;
+ *)
+ usage
+ ;;
+ esac
+done
+shift $((OPTIND-1))
+
+# get some common useful helpers for krb5
+. /root/client_helper.sh
+
+# Get kerberos credentials for user1
+admin_kinit
+admin_klist > /dev/null 2>&1 || die "Cannot get kerberos credentials for user ${USER}"
+
+echo "Generation of a repack report"
+
+DATE=`date +%d-%m-%y-%H:%M:%S`
+
+ARCHIVE_FILE_LS_RESULT_PATH=${REPORT_DIRECTORY}/${VID}_af_ls_${DATE}.json
+NOT_REPACKED_JSON_PATH=${REPORT_DIRECTORY}/${VID}_report_not_repacked_${DATE}.json
+SELF_REPACKED_JSON_PATH=${REPORT_DIRECTORY}/${VID}_report_self_repacked_${DATE}.json
+REPACKED_JSON_PATH=${REPORT_DIRECTORY}/${VID}_report_repacked_${DATE}.json
+
+
+echo "1. Generate archive file ls result into ${ARCHIVE_FILE_LS_RESULT_PATH} file..."
+admin_cta --json archivefile ls --vid ${VID} > ${ARCHIVE_FILE_LS_RESULT_PATH}
+echo "OK"
+
+echo "2. Generate the non-repacked files report into ${NOT_REPACKED_JSON_PATH} file..."
+jq -r '[.[] | select(.tf.supersededByVid == "")]' ${ARCHIVE_FILE_LS_RESULT_PATH} > ${NOT_REPACKED_JSON_PATH}
+echo "OK"
+
+echo "3. Generating the self-repacked files report into ${SELF_REPACKED_JSON_PATH} file..."
+jq -r '[.[] | select((.tf.supersededByVid == .tf.vid) and (.tf.fSeq < .tf.supersededByFSeq))]' ${ARCHIVE_FILE_LS_RESULT_PATH} > ${SELF_REPACKED_JSON_PATH}
+echo "OK"
+
+echo "4. Generate the repacked files report into ${REPACKED_JSON_PATH} file..."
+jq -r '[.[] | select((.tf.supersededByVid != "") and (.tf.supersededByVid != .tf.vid))]' ${ARCHIVE_FILE_LS_RESULT_PATH} > ${REPACKED_JSON_PATH}
+echo "OK"
+
+echo "5. Report of the repacked tape"
+echo
+NB_NON_REPACKED_FILES=$(jq '[.[]] | length' ${NOT_REPACKED_JSON_PATH} || 0)
+echo "Number of non repacked files : ${NB_NON_REPACKED_FILES}"
+if [ ${NB_NON_REPACKED_FILES} -ne 0 ]
+then
+ header="ArchiveID\tFSeq\tSize"
+ { echo -e $header; jq -r '.[] | [.af.archiveId,.tf.fSeq, .af.size] | @tsv' ${NOT_REPACKED_JSON_PATH}; } | column -t
+fi;
+echo
+NB_SELF_REPACKED_FILES=$(jq '[.[]] | length' ${SELF_REPACKED_JSON_PATH} || 0)
+echo "Number of self-repacked files : ${NB_SELF_REPACKED_FILES}"
+if [ ${NB_SELF_REPACKED_FILES} -ne 0 ]
+then
+ header="ArchiveID\tFSeq\tSize"
+ { echo -e $header; jq -r '.[] | [.af.archiveId, .tf.fSeq, .af.size] | @tsv' ${SELF_REPACKED_JSON_PATH}; } | column -t
+fi;
+echo
+NB_REPACKED_FILES=$(jq '[.[]] | length' ${REPACKED_JSON_PATH} || 0)
+echo "Number of repacked files : ${NB_REPACKED_FILES}"
+if [ ${NB_REPACKED_FILES} -ne 0 ]
+then
+ header="DestinationVID\tNbFiles\ttotalSize\n"
+ { echo -e $header; jq -r 'group_by(.tf.supersededByVid)[] | [(.[0].tf.supersededByVid),([.[] | .tf.supersededByFSeq] | length),(reduce [.[] | .af.size | tonumber][] as $currentSize (0; . + $currentSize))] | @tsv' ${REPACKED_JSON_PATH}; } | column -t
+fi;
+
+echo "End of the repack report"
\ No newline at end of file
diff --git a/continuousintegration/orchestration/tests/repack_systemtest.sh b/continuousintegration/orchestration/tests/repack_systemtest.sh
index 306ef4cf0028644134257af006d0e0e0c8b0b58f..f6a911190cd1fa28ef017e4afbfa4f43639d2b68 100755
--- a/continuousintegration/orchestration/tests/repack_systemtest.sh
+++ b/continuousintegration/orchestration/tests/repack_systemtest.sh
@@ -22,12 +22,15 @@ exit 1
testRepackBufferURL(){
echo "Testing the repack buffer URL at root://${EOSINSTANCE}/${REPACK_BUFFER_BASEDIR}"
- eos root://${EOSINSTANCE} ls -d ${REPACK_BUFFER_BASEDIR} || die "Repack bufferURL directory does not exist"
+ eos root://${EOSINSTANCE} ls -d ${REPACK_BUFFER_BASEDIR} 1> /dev/null || die "Repack bufferURL directory does not exist"
echo "Testing the insertion of a test file in the buffer URL"
tempFilePath=$(mktemp /tmp/testFile.XXXX)
tempFileName=${tempFilePath##*/}
xrdcp ${tempFilePath} ${FULL_REPACK_BUFFER_URL}/${tempFileName} || die "Unable to write a file into the repack buffer directory"
- echo "OK"
+ echo "File ${tempFilePath} written in ${FULL_REPACK_BUFFER_URL}/${tempFileName}"
+ echo "Deleting test file from the test directory"
+ eos root://${EOSINSTANCE} rm ${REPACK_BUFFER_BASEDIR}/${tempFileName} || die "Unable to delete the testing file"
+ echo "Test repack buffer URL OK"
}
if [ $# -lt 2 ]
@@ -47,7 +50,7 @@ while getopts "v:e:b:t:" o; do
REPACK_BUFFER_BASEDIR=${OPTARG}
;;
t)
- WAIT_FOR_REPACK_TIMEOUT={$OPTARG}
+ WAIT_FOR_REPACK_TIMEOUT=${OPTARG}
;;
*)
usage
@@ -82,9 +85,6 @@ admin_cta repack rm --vid ${VID_TO_REPACK}
echo "Marking the tape ${VID_TO_REPACK} as full before Repacking it"
admin_cta tape ch --vid ${VID_TO_REPACK} --full true
-echo "State of the tape VID ${VID_TO_REPACK} BEFORE repack"
-admin_cta --json tape ls --vid ${VID_TO_REPACK} | jq .
-
echo "Launching repack request for VID ${VID_TO_REPACK}, bufferURL = ${FULL_REPACK_BUFFER_URL}"
admin_cta re add --vid ${VID_TO_REPACK} --justmove --bufferurl ${FULL_REPACK_BUFFER_URL}
@@ -104,5 +104,4 @@ if test 1 = `admin_cta repack ls --vid ${VID_TO_REPACK} | grep -E "Failed" | wc
exit 1
fi
-echo "State of the tape VID ${VID_TO_REPACK} AFTER repack"
-admin_cta --json tape ls --vid ${VID_TO_REPACK} | jq .
\ No newline at end of file
+exec /root/repack_generate_report.sh -v ${VID_TO_REPACK}
\ No newline at end of file
diff --git a/continuousintegration/orchestration/tests/repack_systemtest_wrapper.sh b/continuousintegration/orchestration/tests/repack_systemtest_wrapper.sh
index fe31c8ce4223115bdbb3523e4775ac1b9acd37c3..dabdec972d5e8a11b3c42689459d34b442cf0705 100755
--- a/continuousintegration/orchestration/tests/repack_systemtest_wrapper.sh
+++ b/continuousintegration/orchestration/tests/repack_systemtest_wrapper.sh
@@ -49,6 +49,7 @@ kubectl -n ${NAMESPACE} exec ctaeos -- eos chmod 1777 ${REPACK_BUFFER_URL}
source ./repack_helper.sh
kubectl -n ${NAMESPACE} cp repack_systemtest.sh client:/root/repack_systemtest.sh
+kubectl -n ${NAMESPACE} cp repack_generate_report.sh client:/root/repack_generate_report.sh
echo
echo "Launching a round trip repack request"
diff --git a/mediachanger/castorrmc/rmc/cta-smc.1cta b/mediachanger/castorrmc/rmc/cta-smc.1cta
index 04002012116449e0d2032bd0d672b79a051b0650..b72afc7a5120d7ade1a27a4c911f10376836aa6b 100644
--- a/mediachanger/castorrmc/rmc/cta-smc.1cta
+++ b/mediachanger/castorrmc/rmc/cta-smc.1cta
@@ -24,32 +24,22 @@ cta-smc \- command line interface to drive robotic devices through SCSI
.BI -D " drive_ordinal"
[
.BI -V " vid"
-] [
-.BI -v
-]
+]
.br
.B cta-smc
.BI -e
.BI -V " vid"
-[
-.BI -v
-]
.br
.B cta-smc
.BI -i
[
.BI -V " vid"
-] [
-.BI -v
-]
+]
.br
.B cta-smc
.BI -m
.BI -D " drive_ordinal"
.BI -V " vid"
-[
-.BI -v
-]
.br
.B cta-smc
.B -q D
@@ -57,19 +47,19 @@ cta-smc \- command line interface to drive robotic devices through SCSI
.BI -D " drive_ordinal"
]
[
-.BI -v
+.BI -j
]
.br
.B cta-smc
.B -q L
[
-.BI -v
+.BI -j
]
.br
.B cta-smc
.B -q P
[
-.BI -v
+.BI -j
]
.br
.B cta-smc
@@ -79,7 +69,7 @@ cta-smc \- command line interface to drive robotic devices through SCSI
] [
.BI -S " starting_slot"
] [
-.BI -v
+.BI -j
]
.br
.B cta-smc
@@ -89,7 +79,7 @@ cta-smc \- command line interface to drive robotic devices through SCSI
] [
.BI -V " vid"
] [
-.BI -v
+.BI -j
]
.SH DESCRIPTION
.B cta-smc
@@ -161,8 +151,8 @@ specifies the starting slot address for the query operation.
A full vid or a pattern may be specified. In the latter case wild card
characters '*' and '?' may be used but must be escaped because of the shell.
.TP
-.B \-v
-verbose mode. On query requests it prints headers on top of the columns.
+.B \-j
+script mode. On query requests it prints json.
.SH EXAMPLES
.LP
@@ -195,7 +185,7 @@ Device Count = 2, Start = 1030
To query the status of all the drives:
.br
.RS
-.B "cta-smc -q D -v"
+.B "cta-smc -q D"
.sp
.nf
.cs R 20
@@ -209,7 +199,7 @@ Drive Ordinal Element Addr. Status Vid
To get the list of a few slots in the library:
.br
.RS
-.B "cta-smc -q S -S 20 -N 10 -v"
+.B "cta-smc -q S -S 20 -N 10"
.sp
.nf
.cs R 20
@@ -231,7 +221,7 @@ Element Addr. Vid
To get the status of volumes for which the vid starts with JK200
.br
.RS
-.B "cta-smc -q V -V 'JK200*' -v"
+.B "cta-smc -q V -V 'JK200*'"
.sp
.nf
.cs R 20
diff --git a/mediachanger/castorrmc/rmc/smc.c b/mediachanger/castorrmc/rmc/smc.c
index 54b229003d5055b6837df2852917423bb1d2b745..52669b475fe4a908d646f3e40d27e47da7fe5b5b 100644
--- a/mediachanger/castorrmc/rmc/smc.c
+++ b/mediachanger/castorrmc/rmc/smc.c
@@ -14,12 +14,15 @@
#include "serrno.h"
#include "smc_constants.h"
#include "getconfent.h"
+#include "getopt.h"
#include <ctype.h>
/* exit codes */
#define USERR 1
#define PATH_CONF "cta-smc.conf"
+#define TEXT_RED "\x1b[31;1m"
+#define TEXT_NORMAL "\x1b[0m"
extern char *optarg;
@@ -33,30 +36,80 @@ static void smc_str_upper(char *const s) {
static void smc_usage(const char *const cmd)
{
- fprintf (stderr, "usage: %s ", cmd);
+ fprintf (stderr, "Usage:\n");
fprintf (stderr,
- "-d -D drive_ordinal [-V vid] [-v]\n"
- "\t-e -V vid [-v]\n"
- "\t-i [-V vid] [-v]\n"
- "\t-m -D drive_ordinal -V vid [-v]\n"
- "\t-q D [-D drive_ordinal] [-v]\n"
- "\t-q L [-v]\n"
- "\t-q S [-N nbelem] [-S starting_slot] [-v]\n"
- "\t-q V [-N nbelem] [-V vid] [-v]\n");
+ " %s -d -D drive_ordinal [-V vid]\n"
+ " %s -e -V vid\n"
+ " %s -i [-V vid]\n"
+ " %s -m -D drive_ordinal -V vid\n"
+ " %s -q D [-D drive_ordinal] [-j]\n"
+ " %s -q L [-j]\n"
+ " %s -q P [-j]\n"
+ " %s -q S [-N nbelem] [-S starting_slot] [-j]\n"
+ " %s -q V [-N nbelem] [-V vid] [-j]\n",
+ cmd, cmd, cmd, cmd, cmd, cmd, cmd, cmd, cmd);
}
+void smc_qdrive_humanPrint(const struct robot_info *const robot_info,
+ const struct smc_element_info *const element_info, const int numberOfElements,
+ const int useSpectraLib) {
+ char *pstatus;
+ int i;
+ printf (TEXT_RED "Drive Ordinal\tElement Addr.\t Status Vid\n" TEXT_NORMAL);
+ for (i = 0; i < numberOfElements; i++) {
+ if (((element_info+i)->state & 0x1) == 0)
+ pstatus = "free";
+ else if ((element_info+i)->state & 0x4)
+ pstatus = "error";
+ else if ((element_info+i)->state & 0x8 && !useSpectraLib)
+ pstatus = "unloaded";
+ else
+ pstatus = "loaded";
+ printf ("%13d\t%13d\t%8s %s\n",
+ (element_info+i)->element_address-robot_info->device_start,
+ (element_info+i)->element_address, pstatus,
+ (element_info+i)->name);
+ }
+}
+void smc_qdrive_jsonPrint(const struct robot_info *const robot_info,
+ const struct smc_element_info *const element_info, const int numberOfElements,
+ const int useSpectraLib) {
+ char *pstatus;
+ int i;
+ printf ("[");
+ for (i = 0; i < numberOfElements; i++) {
+ if (((element_info+i)->state & 0x1) == 0)
+ pstatus = "free";
+ else if ((element_info+i)->state & 0x4)
+ pstatus = "error";
+ else if ((element_info+i)->state & 0x8 && !useSpectraLib)
+ pstatus = "unloaded";
+ else
+ pstatus = "loaded";
+ if (0 != i) {
+ printf(",");
+ }
+ printf ("{\"driveOrdinal\":%d,"
+ "\"elementAddress\":%d,"
+ "\"status\":\"%s\","
+ "\"vid\":\"%s\"}",
+ (element_info+i)->element_address-robot_info->device_start,
+ (element_info+i)->element_address,
+ pstatus,
+ (element_info+i)->name);
+ }
+ printf ("]");
+}
static int smc_qdrive (
const char *const rmchost,
const int fd,
const struct robot_info *const robot_info,
int drvord,
- const int verbose)
+ const int isJsonEnabled)
{
int c;
struct smc_element_info *element_info;
- int i;
int nbelem;
- char *pstatus;
char *smcLibraryType;
char useSpectraLib;
@@ -75,59 +128,120 @@ static int smc_qdrive (
free (element_info);
return (c);
}
- if (verbose)
- printf ("Drive Ordinal\tElement Addr.\tStatus\t\tVid\n");
-
useSpectraLib=0;
smcLibraryType = getconfent_fromfile(PATH_CONF,"SMC","LIBRARY_TYPE",0);
if (NULL != smcLibraryType &&
0 == strcasecmp(smcLibraryType,"SPECTRA")) {
useSpectraLib = 1;
}
-
- for (i = 0; i < c; i++) {
- if (((element_info+i)->state & 0x1) == 0)
- pstatus = "free";
- else if ((element_info+i)->state & 0x4)
- pstatus = "error";
- else if ((element_info+i)->state & 0x8 && !useSpectraLib)
- pstatus = "unloaded";
- else
- pstatus = "loaded";
- printf (" %2d\t %d\t%s\t%s\n",
- (element_info+i)->element_address-robot_info->device_start,
- (element_info+i)->element_address, pstatus,
- (element_info+i)->name);
- }
+ if (isJsonEnabled) {
+ smc_qdrive_jsonPrint(robot_info, element_info, c, useSpectraLib);
+ } else {
+ smc_qdrive_humanPrint(robot_info, element_info, c, useSpectraLib);
+ }
free (element_info);
return (0);
}
-static int smc_qlib (const struct robot_info *const robot_info)
+void smc_qlib_humanPrint(const struct robot_info *const robot_info) {
+ printf ("Vendor/Product/Revision = <%s>\n", robot_info->inquiry);
+ printf ("Transport Count = %d, Start = %d\n",
+ robot_info->transport_count, robot_info->transport_start);
+ printf ("Slot Count = %d, Start = %d\n",
+ robot_info->slot_count, robot_info->slot_start);
+ printf ("Port Count = %d, Start = %d\n",
+ robot_info->port_count, robot_info->port_start);
+ printf ("Device Count = %d, Start = %d\n",
+ robot_info->device_count, robot_info->device_start);
+}
+
+void smc_qlib_jsonPrint(const struct robot_info *const robot_info) {
+ char T10Vendor[9];
+ char prodId[17];
+ char prodRevLvl[5];
+ memcpy (T10Vendor, robot_info->inquiry, 8);
+ T10Vendor[8] = '\0';
+ memcpy(prodId, robot_info->inquiry + 8, 16);
+ prodId[16] = '\0';
+ memcpy(prodRevLvl,robot_info->inquiry + 8 + 16, 4);
+ prodRevLvl[4] = '\0';
+ printf ("[");
+ printf ("{\"inquiry\":{\"vendor\":\"%s\",\"product\":\"%s\",\"revision\":\"%s\"},",
+ T10Vendor, prodId, prodRevLvl);
+ printf ("\"transport\":{\"count\":%d,\"start\":%d},",
+ robot_info->transport_count, robot_info->transport_start);
+ printf ("\"slot\":{\"count\":%d,\"start\":%d},",
+ robot_info->slot_count, robot_info->slot_start);
+ printf ("\"port\":{\"count\":%d,\"start\":%d},",
+ robot_info->port_count, robot_info->port_start);
+ printf ("\"device\":{\"count\":%d,\"start\":%d}",
+ robot_info->device_count, robot_info->device_start);
+ printf ("}]");
+}
+
+static int smc_qlib (const struct robot_info *const robot_info,
+ const int isJsonEnabled)
{
- printf ("Vendor/Product/Revision = <%s>\n", robot_info->inquiry);
- printf ("Transport Count = %d, Start = %d\n",
- robot_info->transport_count, robot_info->transport_start);
- printf ("Slot Count = %d, Start = %d\n",
- robot_info->slot_count, robot_info->slot_start);
- printf ("Port Count = %d, Start = %d\n",
- robot_info->port_count, robot_info->port_start);
- printf ("Device Count = %d, Start = %d\n",
- robot_info->device_count, robot_info->device_start);
+ if (isJsonEnabled) {
+ smc_qlib_jsonPrint(robot_info);
+ } else {
+ smc_qlib_humanPrint(robot_info);
+ }
return (0);
}
+void smc_qport_humanPrint(const struct smc_element_info *const element_info,
+ const int numberOfElements) {
+ char *pstatus;
+ int i;
+ printf (TEXT_RED "Element Addr.\tVid\tImpExp\n" TEXT_NORMAL);
+ for (i = 0; i < numberOfElements; i++) {
+ if (((element_info+i)->state & 0x1) == 0)
+ pstatus = "";
+ else if (((element_info+i)->state & 0x2) == 0)
+ pstatus = "export";
+ else
+ pstatus = "import";
+ printf (" %4d\t%s\t%s\n",
+ (element_info+i)->element_address,
+ (element_info+i)->name, pstatus);
+ }
+}
+
+void smc_qport_jsonPrint(const struct smc_element_info *const element_info,
+ const int numberOfElements) {
+ char *pstatus;
+ int i;
+ printf ("[");
+ for (i = 0; i < numberOfElements; i++) {
+ if (((element_info+i)->state & 0x1) == 0)
+ pstatus = "";
+ else if (((element_info+i)->state & 0x2) == 0)
+ pstatus = "export";
+ else
+ pstatus = "import";
+ if (0 != i) {
+ printf(",");
+ }
+ printf ("{\"elementAddress\":%d,"
+ "\"vid\":\"%s\","
+ "\"state\":\"%s\"}",
+ (element_info+i)->element_address,
+ (element_info+i)->name,
+ pstatus);
+ }
+ printf ("]");
+}
+
static int smc_qport (
const char *const rmchost,
const int fd,
const struct robot_info *const robot_info,
- const int verbose)
+ const int isJsonEnabled)
{
int c;
struct smc_element_info *element_info;
- int i;
int nbelem;
- char *pstatus;
nbelem = robot_info->port_count;
if ((element_info = malloc (nbelem * sizeof(struct smc_element_info))) == NULL) {
@@ -140,34 +254,51 @@ static int smc_qport (
free (element_info);
return (serrno - ERMCRBTERR);
}
- if (verbose)
- printf ("Element Addr.\tVid\tImpExp\n");
- for (i = 0; i < c; i++) {
- if (((element_info+i)->state & 0x1) == 0)
- pstatus = "";
- else if (((element_info+i)->state & 0x2) == 0)
- pstatus = "export";
- else
- pstatus = "import";
- printf (" %4d\t%s\t%s\n",
- (element_info+i)->element_address,
- (element_info+i)->name, pstatus);
+ if (isJsonEnabled) {
+ smc_qport_jsonPrint(element_info, c);
+ } else {
+ smc_qport_humanPrint(element_info, c);
}
free (element_info);
return (0);
}
-
+
+void smc_qslot_humanPrint(const struct smc_element_info *element_info,
+ const int numberOfElements) {
+ int i;
+ printf (TEXT_RED "Element Addr.\tVid\n" TEXT_NORMAL);
+ for (i = 0; i < numberOfElements; i++) {
+ printf (" %4d\t%s\n",
+ element_info[i].element_address, element_info[i].name);
+ }
+}
+
+void smc_qslot_jsonPrint(const struct smc_element_info *element_info,
+ const int numberOfElements) {
+ int i;
+ printf ("[");
+ for (i = 0; i < numberOfElements; i++) {
+ if (0 != i) {
+ printf(",");
+ }
+ printf ("{\"elementAddress\":%4d,"
+ "\"vid\":\"%s\"}",
+ element_info[i].element_address,
+ element_info[i].name);
+ }
+ printf ("]");
+}
+
static int smc_qslot (
const char *const rmchost,
const int fd,
const struct robot_info *robot_info,
int slotaddr,
int nbelem,
- const int verbose)
+ const int isJsonEnabled)
{
int c;
struct smc_element_info *element_info;
- int i;
if (nbelem == 0) {
if (slotaddr < 0)
@@ -187,29 +318,74 @@ static int smc_qslot (
free (element_info);
return (serrno - ERMCRBTERR);
}
- if (verbose)
- printf ("Element Addr.\tVid\n");
- for (i = 0; i < c; i++) {
- printf (" %4d\t%s\n",
- element_info[i].element_address, element_info[i].name);
- }
+ if (isJsonEnabled) {
+ smc_qslot_jsonPrint(element_info, c);
+ } else {
+ smc_qslot_humanPrint(element_info, c);
+ }
free (element_info);
return (0);
}
+void smc_qvid_humanPrint(const struct smc_element_info *const element_info,
+ const int numberOfElements){
+ int i;
+ char *ptype;
+ char ptypes[5][6] = {"", "hand", "slot", "port", "drive"};
+ printf (TEXT_RED "Vid\tElement Addr.\tElement Type\n" TEXT_NORMAL);
+ for (i = 0; i < numberOfElements; i++) {
+ ptype = ptypes[(element_info+i)->element_type];
+ if ((element_info+i)->element_type == 3) {
+ if (((element_info+i)->state & 0x2) == 0) {
+ ptype = "export";
+ } else {
+ ptype = "import";
+ }
+ }
+ printf ("%s\t %4d\t%s\n",
+ (element_info+i)->name, (element_info+i)->element_address,
+ ptype);
+ }
+}
+
+void smc_qvid_jsonPrint(const struct smc_element_info *const element_info,
+ const int numberOfElements){
+ int i;
+ char *ptype;
+ char ptypes[5][6] = {"", "hand", "slot", "port", "drive"};
+ printf ("[");
+ for (i = 0; i < numberOfElements; i++) {
+ ptype = ptypes[(element_info+i)->element_type];
+ if ((element_info+i)->element_type == 3) {
+ if (((element_info+i)->state & 0x2) == 0) {
+ ptype = "export";
+ } else {
+ ptype = "import";
+ }
+ }
+ if (0 != i) {
+ printf(",");
+ }
+ printf ("{\"vid\":\"%s\","
+ "\"elementAddress\":%d,"
+ "\"elementType\":\"%s\"}",
+ (element_info+i)->name,
+ (element_info+i)->element_address,
+ ptype);
+ }
+ printf ("]");
+}
+
static int smc_qvid (
const char *const rmchost,
const int fd,
const struct robot_info *const robot_info,
const char *reqvid,
int nbelem,
- const int verbose)
+ const int isJsonEnabled)
{
int c;
struct smc_element_info *element_info;
- int i;
- char *ptype;
- static char ptypes[5][6] = {"", "hand", "slot", "port", "drive"};
const char *vid;
if (*reqvid)
@@ -233,21 +409,11 @@ static int smc_qvid (
free (element_info);
return (serrno - ERMCRBTERR);
}
- if (verbose)
- printf ("Vid\tElement Addr.\tElement Type\n");
- for (i = 0; i < c; i++) {
- ptype = ptypes[(element_info+i)->element_type];
- if ((element_info+i)->element_type == 3) {
- if (((element_info+i)->state & 0x2) == 0) {
- ptype = "export";
- } else {
- ptype = "import";
- }
- }
- printf ("%s\t %4d\t%s\n",
- (element_info+i)->name, (element_info+i)->element_address,
- ptype);
- }
+ if (isJsonEnabled) {
+ smc_qvid_jsonPrint(element_info, c);
+ } else {
+ smc_qvid_humanPrint(element_info, c);
+ }
free (element_info);
return (0);
}
@@ -268,14 +434,25 @@ int main(const int argc,
struct robot_info robot_info;
const char *rmchost = "localhost";
int slotaddr = -1;
- int targetslotaddr = -1;
- int verbose = 0;
char vid[7];
+ int isJsonEnabled = 0;
/* parse and check command options */
-
+ struct option longopts [] = {
+ {"drive", required_argument, NULL, 'D'},
+ {"dismount", no_argument, NULL, 'd'},
+ {"export", no_argument, NULL, 'e'},
+ {"import", no_argument, NULL, 'i'},
+ {"mount", no_argument, NULL, 'm'},
+ {"nbelem", required_argument, NULL, 'N'},
+ {"query", required_argument, NULL, 'q'},
+ {"slot", required_argument, NULL, 'S'},
+ {"vid", required_argument, NULL, 'V'},
+ {"json", no_argument, NULL, 'j'},
+ {NULL, 0, NULL, 0}
+ };
memset (vid, '\0', sizeof(vid));
- while ((c = getopt (argc, argv, "D:deimN:q:S:V:vT:")) != EOF) {
+ while ((c = getopt_long(argc, argv, "D:deimN:q:S:V:j", longopts, NULL)) != EOF) {
switch (c) {
case 'D': /* drive ordinal */
drvord = strtol (optarg, &dp, 10);
@@ -335,13 +512,6 @@ int main(const int argc,
errflg++;
}
break;
- case 'T': /* Target slot */
- targetslotaddr = strtol (optarg, &dp, 10);
- if (*dp != '\0' || targetslotaddr < 0) {
- fprintf (stderr, SR001);
- errflg++;
- }
- break;
case 'V': /* vid */
n = strlen (optarg);
if (n > 6) {
@@ -352,9 +522,9 @@ int main(const int argc,
smc_str_upper (vid);
}
break;
- case 'v':
- verbose = 1;
- break;
+ case 'j':
+ isJsonEnabled = 1;
+ break;
case '?':
errflg++;
break;
@@ -422,21 +592,21 @@ int main(const int argc,
switch (qry_type) {
case 'D':
c = smc_qdrive (rmchost, fd, &robot_info, drvord,
- verbose);
+ isJsonEnabled);
break;
case 'L':
- c = smc_qlib (&robot_info);
+ c = smc_qlib (&robot_info, isJsonEnabled);
break;
case 'P':
- c = smc_qport (rmchost, fd, &robot_info, verbose);
+ c = smc_qport (rmchost, fd, &robot_info, isJsonEnabled);
break;
case 'S':
c = smc_qslot (rmchost, fd, &robot_info, slotaddr,
- nbelem, verbose);
+ nbelem, isJsonEnabled);
break;
case 'V':
c = smc_qvid (rmchost, fd, &robot_info, vid,
- nbelem, verbose);
+ nbelem, isJsonEnabled);
break;
}
break;
diff --git a/objectstore/RepackRequest.cpp b/objectstore/RepackRequest.cpp
index 0a0bb02505c8e06d831ad49b49f3379e75fd3668..a61e58e3097ccd4fb06f8c7ac80acb7be77caa31 100644
--- a/objectstore/RepackRequest.cpp
+++ b/objectstore/RepackRequest.cpp
@@ -138,6 +138,8 @@ common::dataStructures::RepackInfo RepackRequest::getInfo() {
ret.failedBytesToArchive = m_payload.failedtoarchivebytes();
ret.failedFilesToRetrieve = m_payload.failedtoretrievefiles();
ret.failedBytesToRetrieve = m_payload.failedtoretrievebytes();
+ ret.archivedFiles = m_payload.archivedfiles();
+ ret.retrievedFiles = m_payload.retrievedfiles();
ret.lastExpandedFseq = m_payload.lastexpandedfseq();
ret.userProvidedFiles = m_payload.userprovidedfiles();
ret.isExpandFinished = m_payload.is_expand_finished();
@@ -194,9 +196,9 @@ void RepackRequest::setStatus(){
return;
}
}
- //Expand is finished or not, if we have retrieved files, we are in Running,
+ //Expand is finished or not, if we have retrieved files or not (first reporting), we are in Running,
//else we are in starting
- if(m_payload.retrievedfiles()){
+ if(m_payload.retrievedfiles() || m_payload.failedtoretrievefiles()){
setStatus(common::dataStructures::RepackInfo::Status::Running);
} else {
setStatus(common::dataStructures::RepackInfo::Status::Starting);
diff --git a/python/eosfstgcd/cta-fst-gcd b/python/eosfstgcd/cta-fst-gcd
index 1c579306e3c72dce84be5a228527cbd587525303..358a983c12c62b545a02a36194028946e9db0f19 100755
--- a/python/eosfstgcd/cta-fst-gcd
+++ b/python/eosfstgcd/cta-fst-gcd
@@ -100,10 +100,10 @@ class Gc:
logging.config.dictConfig(config)
def configurereallogging(self):
- if None == self.logfilepath:
+ if None == self.logfile:
raise Exception("Cannot configure file based logging because the log file path has not been set")
- loggingdir = os.path.dirname(self.logfilepath)
+ loggingdir = os.path.dirname(self.logfile)
if not os.path.isdir(loggingdir):
raise UserError("The logging directory {} is not a directory or does not exist".format(loggingdir))
if not os.access(loggingdir, os.W_OK):
@@ -124,7 +124,7 @@ class Gc:
'level': 'INFO',
'formatter': 'stdout',
'class': 'logging.handlers.TimedRotatingFileHandler',
- 'filename' : self.logfilepath,
+ 'filename' : self.logfile,
'when' : 'midnight'
}
},
@@ -142,16 +142,16 @@ class Gc:
raise UserError(err)
def configurelogging(self):
- if None == self.logfilepath:
+ if None == self.logfile:
self.configuredummylogging()
else:
self.configurereallogging()
- def __init__(self, env, logfilepath = '/var/log/eos/fst/cta-fst-gcd.log'):
+ def __init__(self, env, logfile):
self.programname = 'cta-fst-gcd'
self.env = env
self.conffilepath = '/etc/cta/cta-fst-gcd.conf'
- self.logfilepath = logfilepath
+ self.logfile = logfile
self.fqdn = socket.getfqdn()
self.localfilesystempaths = []
self.nbfilesconsideredsincelastreport = 0
@@ -188,7 +188,7 @@ class Gc:
return result
- def eosstagerrm(self, fxid):
+ def eosstagerrm(self, fxid, subdir, freebytes):
logger = logging.getLogger('gc')
mgmurl = "root://{}".format(self.mgmhost)
cmd = "eos {} stagerrm fxid:{}".format(mgmurl, fxid)
@@ -197,7 +197,7 @@ class Gc:
process = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
stdout,stderr = process.communicate()
if 0 == process.returncode:
- logger.info("Executed {}".format(cmd))
+ logger.info("minfreebytes={} subdir={} freebytesbefore={} executed='{}'".format(self.minfreebytes, subdir, freebytes, cmd))
def processfile(self, subdir, fstfile):
statvfs = os.statvfs(subdir)
@@ -210,7 +210,7 @@ class Gc:
now = time.time()
agesecs = now - statinfo.st_ctime
if agesecs > self.gcagesecs:
- self.eosstagerrm(fstfile)
+ self.eosstagerrm(fstfile, subdir, freebytes)
self.nbfilesconsideredsincelastreport = self.nbfilesconsideredsincelastreport + 1
if self.nbfilesbeforereport == self.nbfilesconsideredsincelastreport:
@@ -304,12 +304,11 @@ class Gc:
time.sleep(sleeptime)
def main():
- programname = 'cta-fst-gcd'
-
parser = argparse.ArgumentParser()
+ parser.add_argument("-l", "--logfile", default="/var/log/eos/fst/cta-fst-gcd.log", help="Log file path")
args = parser.parse_args()
- gc = Gc(os.environ)
+ gc = Gc(env=os.environ, logfile=args.logfile)
try:
gc.run()
diff --git a/python/eosfstgcd/cta-fst-gcd.1cta b/python/eosfstgcd/cta-fst-gcd.1cta
index 046af3f0af166f626a2f58be19e723daa2267478..1ababe513f20ea1037d715374ea53f6b66defa01 100644
--- a/python/eosfstgcd/cta-fst-gcd.1cta
+++ b/python/eosfstgcd/cta-fst-gcd.1cta
@@ -39,6 +39,9 @@ filename and then running \fBeos stagerm fxid:<fid-hex>\fP.
.TP
\fB\-h, \-\-help
Prints the usage message.
+.TP
+\fB\-l LOGFILE, \-\-logfile LOGFILE
+Sets the path of the log file to \fBLOGFILE\fP. If not set then the default value of \fB/var/log/eos/fst/cta-fst-gcd\fP is used.
.SH RETURN VALUE
Zero on success and non-zero on failure.
.SH FILES
@@ -48,7 +51,7 @@ The configuration file of the \fBcta-fst-gcd\fP daemon.
.TP
.B /var/log/eos/fst/cta-fst-gcd.log
-The log file of the \fBcta-fst-gcd\fP daemon.
+The default log file of the \fBcta-fst-gcd\fP daemon. This can be overriden by using the \fB\-l\fP/\fB\-\-logfile\fP option.
.SH AUTHOR
\fBCTA\fP Team
diff --git a/scheduler/ArchiveMount.cpp b/scheduler/ArchiveMount.cpp
index 0cb2df62a402ab24b70cb7cce06e6b23592e2fe1..818f703a145990c1c236c4ac53275f12d361a07c 100644
--- a/scheduler/ArchiveMount.cpp
+++ b/scheduler/ArchiveMount.cpp
@@ -104,16 +104,6 @@ uint32_t cta::ArchiveMount::getNbFiles() const {
return m_dbMount->nbFilesCurrentlyOnTape;
}
-//------------------------------------------------------------------------------
-// checkTapeFSeqForWriting
-//------------------------------------------------------------------------------
-void cta::ArchiveMount::checkTapeFSeqAndDeleteTapeFilesForWriting(uint64_t fSeq) const {
- if(m_catalogue.existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(getVid(),fSeq)){
- throw cta::exception::Exception("Non superseded files have been detected in the tape "+getVid() +" after "+std::to_string(fSeq));
- }
-}
-
-
//------------------------------------------------------------------------------
// createDiskReporter
//------------------------------------------------------------------------------
diff --git a/scheduler/ArchiveMount.hpp b/scheduler/ArchiveMount.hpp
index 509895d54031396f1e61b93a7b3cfb8759ca3831..2073545e4d03613f27ddaf10fb90cf96f5407a18 100644
--- a/scheduler/ArchiveMount.hpp
+++ b/scheduler/ArchiveMount.hpp
@@ -172,14 +172,6 @@ namespace cta {
*/
uint32_t getNbFiles() const override;
- /**
- * Checks wether the writing is possible after the FSeq passed in parameter
- * If TapeFiles are located after FSeq and are not superseded by other TapeFiles
- * we throw an exception : We don't want these files to be lost.
- * @param fSeq : The fSeq after which we want to check if the writing is possible
- */
- void checkTapeFSeqAndDeleteTapeFilesForWriting(uint64_t fSeq) const;
-
/**
* Creates a disk reporter for the ArchiveJob (this is a wrapper).
* @param URL: report address
diff --git a/scheduler/OStoreDB/OStoreDB.cpp b/scheduler/OStoreDB/OStoreDB.cpp
index 825b7a67c217feed73c74ad011c0863d33a233f4..b378b9e31816409b9d17331da43f5b220bbbef7b 100644
--- a/scheduler/OStoreDB/OStoreDB.cpp
+++ b/scheduler/OStoreDB/OStoreDB.cpp
@@ -1591,6 +1591,26 @@ std::unique_ptr<SchedulerDatabase::RepackReportBatch> OStoreDB::getNextRepackRep
return nullptr;
}
+//------------------------------------------------------------------------------
+// OStoreDB::getRepackReportBatches()
+//------------------------------------------------------------------------------
+std::list<std::unique_ptr<SchedulerDatabase::RepackReportBatch>> OStoreDB::getRepackReportBatches(log::LogContext &lc){
+ std::list<std::unique_ptr<SchedulerDatabase::RepackReportBatch>> ret;
+ try{
+ ret.push_back(std::move(getNextSuccessfulRetrieveRepackReportBatch(lc)));
+ } catch (const NoRepackReportBatchFound &){}
+ try{
+ ret.push_back(std::move(getNextFailedRetrieveRepackReportBatch(lc)));
+ } catch (const NoRepackReportBatchFound &){}
+ try{
+ ret.push_back(std::move(getNextSuccessfulArchiveRepackReportBatch(lc)));
+ } catch (const NoRepackReportBatchFound &){}
+ try{
+ ret.push_back(std::move(getNextFailedArchiveRepackReportBatch(lc)));
+ } catch (const NoRepackReportBatchFound &){}
+ return ret;
+}
+
//------------------------------------------------------------------------------
// OStoreDB::getNextSuccessfulRetrieveRepackReportBatch()
//------------------------------------------------------------------------------
diff --git a/scheduler/OStoreDB/OStoreDB.hpp b/scheduler/OStoreDB/OStoreDB.hpp
index a5daab47a625e58bf97a5e412ea38a1073e1d1c4..97c4ad316a05e11bbb030445af97bd28ad731fd9 100644
--- a/scheduler/OStoreDB/OStoreDB.hpp
+++ b/scheduler/OStoreDB/OStoreDB.hpp
@@ -500,6 +500,9 @@ public:
};
std::unique_ptr<SchedulerDatabase::RepackReportBatch> getNextRepackReportBatch(log::LogContext& lc) override;
+
+ std::list<std::unique_ptr<SchedulerDatabase::RepackReportBatch>> getRepackReportBatches(log::LogContext &lc) override;
+
private:
CTA_GENERATE_EXCEPTION_CLASS(NoRepackReportBatchFound);
const size_t c_repackReportBatchSize = 500;
diff --git a/scheduler/OStoreDB/OStoreDBFactory.hpp b/scheduler/OStoreDB/OStoreDBFactory.hpp
index 1e9694ebdcdea1616838228219186b3a00de90c4..b578aedce0b2731c402c5541a803ffffbb9b8912 100644
--- a/scheduler/OStoreDB/OStoreDBFactory.hpp
+++ b/scheduler/OStoreDB/OStoreDBFactory.hpp
@@ -157,6 +157,10 @@ public:
std::unique_ptr<RepackReportBatch> getNextRepackReportBatch(log::LogContext& lc) override {
return m_OStoreDB.getNextRepackReportBatch(lc);
}
+
+ std::list<std::unique_ptr<SchedulerDatabase::RepackReportBatch>> getRepackReportBatches(log::LogContext &lc) override {
+ return m_OStoreDB.getRepackReportBatches(lc);
+ }
JobsFailedSummary getRetrieveJobsFailedSummary(log::LogContext &lc) override {
return m_OStoreDB.getRetrieveJobsFailedSummary(lc);
diff --git a/scheduler/RepackRequestManager.cpp b/scheduler/RepackRequestManager.cpp
index cf5fbf7b6adc9ef497334c9eafbe8a5384c3a218..09c69a09f9f4d02820ee9e4b08bbb566c30b9b3c 100644
--- a/scheduler/RepackRequestManager.cpp
+++ b/scheduler/RepackRequestManager.cpp
@@ -48,8 +48,10 @@ void RepackRequestManager::runOnePass(log::LogContext& lc) {
}
{
- // Do one round of repack subrequest reporting (heavy lifting is done internally).
- m_scheduler.getNextRepackReportBatch(lc).report(lc);
+ // Do all round of repack subrequest reporting (heavy lifting is done internally).
+ for(auto& reportBatch: m_scheduler.getRepackReportBatches(lc)){
+ reportBatch.report(lc);
+ }
}
}
diff --git a/scheduler/Scheduler.cpp b/scheduler/Scheduler.cpp
index 6d6670763e1886d6395321c1db83e82839020916..16f0d76829485c41d2dbda2b76bcd94f366fb4f5 100644
--- a/scheduler/Scheduler.cpp
+++ b/scheduler/Scheduler.cpp
@@ -586,6 +586,16 @@ Scheduler::RepackReportBatch Scheduler::getNextRepackReportBatch(log::LogContext
return ret;
}
+std::list<Scheduler::RepackReportBatch> Scheduler::getRepackReportBatches(log::LogContext &lc){
+ std::list<Scheduler::RepackReportBatch> ret;
+ for(auto& reportBatch: m_db.getRepackReportBatches(lc)){
+ Scheduler::RepackReportBatch report;
+ report.m_DbBatch.reset(reportBatch.release());
+ ret.push_back(std::move(report));
+ }
+ return ret;
+}
+
//------------------------------------------------------------------------------
// Scheduler::RepackReportBatch::report
//------------------------------------------------------------------------------
diff --git a/scheduler/Scheduler.hpp b/scheduler/Scheduler.hpp
index d0e2aa2985b6bd7078d081d91a10966abd082262..d19820ed86e91aa9ab21d2ac6473805ab869f386 100644
--- a/scheduler/Scheduler.hpp
+++ b/scheduler/Scheduler.hpp
@@ -347,6 +347,8 @@ public:
bool empty() { return nullptr == m_DbBatch; }
};
RepackReportBatch getNextRepackReportBatch(log::LogContext & lc);
+ std::list<Scheduler::RepackReportBatch> getRepackReportBatches(log::LogContext &lc);
+
/*======================= Failed archive jobs support ======================*/
SchedulerDatabase::JobsFailedSummary getArchiveJobsFailedSummary(log::LogContext &lc);
diff --git a/scheduler/SchedulerDatabase.hpp b/scheduler/SchedulerDatabase.hpp
index ee1de1f9ba1c223e66c5a0995f4c47411d70b087..c58874553c05aaf371df6380ba44b45f0a96bb4c 100644
--- a/scheduler/SchedulerDatabase.hpp
+++ b/scheduler/SchedulerDatabase.hpp
@@ -492,6 +492,13 @@ public:
*/
virtual std::unique_ptr<RepackReportBatch> getNextRepackReportBatch(log::LogContext & lc) = 0;
+ /**
+ * Return all batches of subrequests from the database to be reported to repack.
+ * @param lc log context
+ * @return the list of all batches to be reported
+ */
+ virtual std::list<std::unique_ptr<RepackReportBatch>> getRepackReportBatches(log::LogContext &lc) = 0;
+
/**
* Set a batch of jobs as reported (modeled on ArchiveMount::setJobBatchSuccessful().
* @param jobsBatch
diff --git a/tapeserver/castor/tape/tapeserver/daemon/DataTransferSessionTest.cpp b/tapeserver/castor/tape/tapeserver/daemon/DataTransferSessionTest.cpp
index 4bd976026c1b9892e871d48d0d9544448af9392c..1326e57d1c8e7d1db82ea51c1747c0ea4fb2dc11 100644
--- a/tapeserver/castor/tape/tapeserver/daemon/DataTransferSessionTest.cpp
+++ b/tapeserver/castor/tape/tapeserver/daemon/DataTransferSessionTest.cpp
@@ -1858,166 +1858,6 @@ TEST_P(DataTransferSessionTest, DataTransferSessionTapeFullOnFlushMigration) {
"mountTotalReadRetries=\"25\" mountTotalWriteRetries=\"25\" mountWriteTransients=\"10\""));
}
-TEST_P(DataTransferSessionTest, WriteDataInTapeWithNonSupersededFilesOnIt) {
- // 0) Prepare the logger for everyone
- cta::log::StringLogger logger("dummy","tapeServerUnitTest",cta::log::DEBUG);
- cta::log::LogContext logContext(logger);
-
- setupDefaultCatalogue();
- // 1) prepare the fake scheduler
- std::string vid = s_vid;
- std::string vid2 = s_vid+"2";
- // cta::MountType::Enum mountType = cta::MountType::RETRIEVE;
-
- // 3) Prepare the necessary environment (logger, plus system wrapper),
- castor::tape::System::mockWrapper mockSys;
- mockSys.delegateToFake();
- mockSys.disableGMockCallsCounting();
- mockSys.fake.setupForVirtualDriveSLC6();
-
- // 4) Create the scheduler
- auto & catalogue = getCatalogue();
- auto & scheduler = getScheduler();
-
- // Always use the same requester
- const cta::common::dataStructures::SecurityIdentity requester("user", "group");
-
- // List to remember the path of each remote file so that the existance of the
- // files can be tested for at the end of the test
- std::list<std::string> remoteFilePaths;
-
- // 5) Create the environment for the migration to happen (library + tape)
- const std::string libraryComment = "Library comment";
- const bool libraryIsDisabled = false;
- catalogue.createLogicalLibrary(s_adminOnAdminHost, s_libraryName,
- libraryIsDisabled, libraryComment);
- {
- auto libraries = catalogue.getLogicalLibraries();
- ASSERT_EQ(1, libraries.size());
- ASSERT_EQ(s_libraryName, libraries.front().name);
- ASSERT_EQ(libraryComment, libraries.front().comment);
- }
- const uint64_t capacityInBytes = 12345678;
- const std::string tapeComment = "Tape comment";
- bool disabled = false;
- bool full = false;
- catalogue.createTape(s_adminOnAdminHost, s_vid, s_mediaType, s_vendor, s_libraryName, s_tapePoolName, capacityInBytes,
- disabled, full, tapeComment);
- catalogue.createTape(s_adminOnAdminHost, vid2, s_mediaType, s_vendor, s_libraryName, s_tapePoolName, capacityInBytes,
- disabled, full, tapeComment);
-
- // Create the mount criteria
- catalogue.createMountPolicy(requester, "immediateMount", 1000, 0, 1000, 0, 1, "Policy comment");
- catalogue.createRequesterMountRule(requester, "immediateMount", s_diskInstance, requester.username, "Rule comment");
-
- //delete is unnecessary
- //pointer with ownership will be passed to the application,
- //which will do the delete
- mockSys.fake.m_pathToDrive["/dev/nst0"] = new castor::tape::tapeserver::drive::FakeDrive();
-
- // We can prepare files for writing on the drive.
- // Tempfiles are in this scope so they are kept alive
- std::vector<std::unique_ptr<unitTests::TempFile>> sourceFiles;
- std::list<uint64_t> archiveFileIds;
- {
- // Label the tape
- castor::tape::tapeFile::LabelSession ls(*mockSys.fake.m_pathToDrive["/dev/nst0"], s_vid, false);
- catalogue.tapeLabelled(s_vid, "T10D6116");
- mockSys.fake.m_pathToDrive["/dev/nst0"]->rewind();
-
- // Create the files and schedule the archivals
- for(int fseq=1; fseq <= 10 ; fseq ++) {
- // Create a source file.
- sourceFiles.emplace_back(cta::make_unique<unitTests::TempFile>());
- sourceFiles.back()->randomFill(1000);
- remoteFilePaths.push_back(sourceFiles.back()->path());
- // Schedule the archival of the file
- cta::common::dataStructures::ArchiveRequest ar;
- ar.checksumType="ADLER32";
- ar.checksumValue=sourceFiles.back()->adler32();
- ar.storageClass=s_storageClassName;
- ar.srcURL=std::string("file://") + sourceFiles.back()->path();
- ar.requester.name = requester.username;
- ar.requester.group = "group";
- ar.fileSize = 1000;
- ar.diskFileID = std::to_string(fseq);
- ar.diskFileInfo.path = "y";
- ar.diskFileInfo.owner = "z";
- ar.diskFileInfo.group = "g";
- const auto archiveFileId = scheduler.checkAndGetNextArchiveFileId(s_diskInstance, ar.storageClass, ar.requester, logContext);
- archiveFileIds.push_back(archiveFileId);
- scheduler.queueArchiveWithGivenId(archiveFileId,s_diskInstance,ar,logContext);
- }
- }
- scheduler.waitSchedulerDbSubthreadsComplete();
- // Report the drive's existence and put it up in the drive register.
- cta::tape::daemon::TpconfigLine driveConfig("T10D6116", "TestLogicalLibrary", "/dev/tape_T10D6116", "manual");
- cta::common::dataStructures::DriveInfo driveInfo;
- driveInfo.driveName=driveConfig.unitName;
- driveInfo.logicalLibrary=driveConfig.logicalLibrary;
- driveInfo.host="host";
- // We need to create the drive in the registry before being able to put it up.
- scheduler.reportDriveStatus(driveInfo, cta::common::dataStructures::MountType::NoMount, cta::common::dataStructures::DriveStatus::Down, logContext);
- scheduler.setDesiredDriveState(s_adminOnAdminHost, driveConfig.unitName, true, false, logContext);
-
- // Create the data transfer session
- DataTransferConfig castorConf;
- castorConf.bufsz = 1024*1024; // 1 MB memory buffers
- castorConf.nbBufs = 10;
- castorConf.bulkRequestRecallMaxBytes = UINT64_C(100)*1000*1000*1000;
- castorConf.bulkRequestRecallMaxFiles = 1000;
- castorConf.bulkRequestMigrationMaxBytes = UINT64_C(100)*1000*1000*1000;
- castorConf.bulkRequestMigrationMaxFiles = 1000;
- castorConf.nbDiskThreads = 1;
- cta::log::DummyLogger dummyLog("dummy", "dummy");
- cta::mediachanger::MediaChangerFacade mc(dummyLog);
- cta::server::ProcessCap capUtils;
- castor::messages::TapeserverProxyDummy initialProcess;
- DataTransferSession sess("tapeHost", logger, mockSys, driveConfig, mc, initialProcess, capUtils, castorConf, scheduler);
- sess.execute();
-
- catalogue.setTapeFull(requester,vid,true);
- //Fake reclaim the tape to reset tape's counters
- catalogue.fakeReclaimTapeForTests(requester,vid);
-
- //Try to write files on tape vid = TstVid
- // Create the files and schedule the archivals
- {
- for(int fseq=1; fseq <= 10 ; fseq ++) {
- // Create a source file.
- sourceFiles.emplace_back(cta::make_unique<unitTests::TempFile>());
- sourceFiles.back()->randomFill(1000);
- remoteFilePaths.push_back(sourceFiles.back()->path());
- // Schedule the archival of the file
- cta::common::dataStructures::ArchiveRequest ar;
- ar.checksumType="ADLER32";
- ar.checksumValue=sourceFiles.back()->adler32();
- ar.storageClass=s_storageClassName;
- ar.srcURL=std::string("file://") + sourceFiles.back()->path();
- ar.requester.name = requester.username;
- ar.requester.group = "group";
- ar.fileSize = 1000;
- ar.diskFileID = std::to_string(fseq);
- ar.diskFileInfo.path = "y";
- ar.diskFileInfo.owner = "z";
- ar.diskFileInfo.group = "g";
- const auto archiveFileId = scheduler.checkAndGetNextArchiveFileId(s_diskInstance, ar.storageClass, ar.requester, logContext);
- archiveFileIds.push_back(archiveFileId);
- scheduler.queueArchiveWithGivenId(archiveFileId,s_diskInstance,ar,logContext);
- }
- }
- scheduler.waitSchedulerDbSubthreadsComplete();
- {
- DataTransferSession sess("tapeHost2", logger, mockSys, driveConfig, mc, initialProcess, capUtils, castorConf, scheduler);
- sess.execute();
- }
- //The session should fail because of the checking of writing in a tape with files that are not superseded
- std::string logToCheck = logger.getLog();
- ASSERT_NE(std::string::npos, logToCheck.find("MSG=\"In MigrationReportPacker::reportEndOfSessionWithErrors(), "
- "pushing a report.\" thread=\"TapeWrite\" tapeDrive=\"T10D6116\" tapeVid=\"TstVid\" mountId=\"2\" "
- "ErrorMesage=\"Non superseded files have been detected in the tape TstVid after 0\" type=\"ReportEndofSessionWithErrors\""));
-}
-
#undef TEST_MOCK_DB
#ifdef TEST_MOCK_DB
static cta::MockSchedulerDatabaseFactory mockDbFactory;
diff --git a/tapeserver/castor/tape/tapeserver/daemon/TapeWriteSingleThread.cpp b/tapeserver/castor/tape/tapeserver/daemon/TapeWriteSingleThread.cpp
index 3258b2458a4c8f9617bf1e2248af65c3299b82d8..f209e4f0df2568daab19f7731b35653012fc47da 100644
--- a/tapeserver/castor/tape/tapeserver/daemon/TapeWriteSingleThread.cpp
+++ b/tapeserver/castor/tape/tapeserver/daemon/TapeWriteSingleThread.cpp
@@ -319,8 +319,6 @@ void castor::tape::tapeserver::daemon::TapeWriteSingleThread::run() {
params.add("capacityInBytes",m_archiveMount.getCapacityInBytes());
m_logContext.log(cta::log::INFO, "Tape session started");
mountTapeReadWrite();
- currentErrorToCount = "Error_tapeFSeqCheckAndTapeFileCleanup";
- m_archiveMount.checkTapeFSeqAndDeleteTapeFilesForWriting(m_lastFseq);
currentErrorToCount = "Error_tapeLoad";
waitForDrive();
currentErrorToCount = "Error_checkingTapeAlert";
diff --git a/xroot_plugins/XrdCtaRepackLs.hpp b/xroot_plugins/XrdCtaRepackLs.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..010fdd29b2c4f83fb628aac6ff7310259b22e274
--- /dev/null
+++ b/xroot_plugins/XrdCtaRepackLs.hpp
@@ -0,0 +1,115 @@
+/**
+ * The CERN Tape Archive (CTA) project
+ * Copyright © 2018 CERN
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <XrdSsiPbOStreamBuffer.hpp>
+#include <catalogue/Catalogue.hpp>
+#include <xrootd/private/XrdSsi/XrdSsiStream.hh>
+
+
+
+namespace cta { namespace xrd {
+ /*!
+ * Stream object which implements "repack ls" command.
+ */
+ class RepackLsStream: public XrdSsiStream {
+ public:
+
+ RepackLsStream(cta::Scheduler& scheduler, const cta::optional<std::string> vid):
+ XrdSsiStream(XrdSsiStream::isActive),m_scheduler(scheduler), m_vid(vid){
+ XrdSsiPb::Log::Msg(XrdSsiPb::Log::DEBUG, LOG_SUFFIX, "RepackLsStream() constructor");
+ if(!vid){
+ m_repackList = m_scheduler.getRepacks();
+ } else {
+ m_repackList.push_back(m_scheduler.getRepack(vid.value()));
+ }
+ }
+
+ virtual ~RepackLsStream(){
+ XrdSsiPb::Log::Msg(XrdSsiPb::Log::DEBUG,LOG_SUFFIX,"~RepackLsStream() destructor");
+ }
+
+ virtual Buffer *GetBuff(XrdSsiErrInfo &eInfo,int &dlen, bool &last) override {
+ XrdSsiPb::Log::Msg(XrdSsiPb::Log::DEBUG, LOG_SUFFIX, "GetBuff(): XrdSsi buffer fill request (", dlen, " bytes)");
+
+ XrdSsiPb::OStreamBuffer<Data> *streambuf;
+
+ try {
+
+ if(m_repackList.empty()) {
+ // Nothing more to send, close the stream
+ last = true;
+ return nullptr;
+ }
+
+ streambuf = new XrdSsiPb::OStreamBuffer<Data>(dlen);
+
+ for(bool is_buffer_full = false; !m_repackList.empty() && !is_buffer_full; m_repackList.pop_front()){
+ Data record;
+ auto &repackRequest = m_repackList.front();
+ auto repackRequestItem = record.mutable_rels_item();
+ repackRequestItem->set_vid(repackRequest.vid);
+ repackRequestItem->set_repack_buffer_url(repackRequest.repackBufferBaseURL);
+ repackRequestItem->set_user_provided_files(repackRequest.userProvidedFiles);
+ repackRequestItem->set_total_files_to_retrieve(repackRequest.totalFilesToRetrieve);
+ repackRequestItem->set_total_bytes_to_retrieve(repackRequest.totalBytesToRetrieve);
+ repackRequestItem->set_total_files_to_archive(repackRequest.totalFilesToArchive);
+ repackRequestItem->set_total_bytes_to_archive(repackRequest.totalBytesToArchive);
+ repackRequestItem->set_retrieved_files(repackRequest.retrievedFiles);
+ repackRequestItem->set_archived_files(repackRequest.archivedFiles);
+ repackRequestItem->set_failed_to_retrieve_files(repackRequest.failedFilesToRetrieve);
+ repackRequestItem->set_failed_to_retrieve_bytes(repackRequest.failedBytesToRetrieve);
+ repackRequestItem->set_failed_to_archive_files(repackRequest.failedFilesToArchive);
+ repackRequestItem->set_failed_to_archive_bytes(repackRequest.failedBytesToArchive);
+ repackRequestItem->set_status(toString(repackRequest.status));
+ //Last expanded fSeq is in reality the next FSeq to Expand. So last one is next - 1
+ repackRequestItem->set_last_expanded_fseq(repackRequest.lastExpandedFseq != 0 ? repackRequest.lastExpandedFseq - 1 : 0);
+ is_buffer_full = streambuf->Push(record);
+ }
+
+ dlen = streambuf->Size();
+ XrdSsiPb::Log::Msg(XrdSsiPb::Log::DEBUG, LOG_SUFFIX, "GetBuff(): Returning buffer with ", dlen, " bytes of data.");
+
+ } catch(cta::exception::Exception &ex) {
+ std::ostringstream errMsg;
+ errMsg << __FUNCTION__ << " failed: Caught CTA exception: " << ex.what();
+ eInfo.Set(errMsg.str().c_str(), ECANCELED);
+ delete streambuf;
+ } catch(std::exception &ex) {
+ std::ostringstream errMsg;
+ errMsg << __FUNCTION__ << " failed: " << ex.what();
+ eInfo.Set(errMsg.str().c_str(), ECANCELED);
+ delete streambuf;
+ } catch(...) {
+ std::ostringstream errMsg;
+ errMsg << __FUNCTION__ << " failed: Caught an unknown exception";
+ eInfo.Set(errMsg.str().c_str(), ECANCELED);
+ delete streambuf;
+ }
+ return streambuf;
+ }
+
+ private:
+ cta::Scheduler &m_scheduler;
+ const cta::optional<std::string> m_vid;
+ std::list<common::dataStructures::RepackInfo> m_repackList;
+ static constexpr const char * const LOG_SUFFIX = "RepackLsStream";
+ };
+
+}}
\ No newline at end of file
diff --git a/xroot_plugins/XrdSsiCtaRequestMessage.cpp b/xroot_plugins/XrdSsiCtaRequestMessage.cpp
index fa2729517d5aec8a7badf2fbffdd5e40f9ba9499..a72b041d0e63fc2fc4d2adbedf50e7bf3102918c 100644
--- a/xroot_plugins/XrdSsiCtaRequestMessage.cpp
+++ b/xroot_plugins/XrdSsiCtaRequestMessage.cpp
@@ -31,7 +31,7 @@ using XrdSsiPb::PbException;
#include "XrdCtaTapePoolLs.hpp"
#include "XrdSsiCtaRequestMessage.hpp"
#include "XrdCtaTapeLs.hpp"
-
+#include "XrdCtaRepackLs.hpp"
namespace cta {
@@ -193,7 +193,7 @@ void RequestMessage::process(const cta::xrd::Request &request, cta::xrd::Respons
processRepack_Rm(request.admincmd(), response);
break;
case cmd_pair(AdminCmd::CMD_REPACK, AdminCmd::SUBCMD_LS):
- processRepack_Ls(request.admincmd(), response);
+ processRepack_Ls(request.admincmd(), response, stream);
break;
case cmd_pair(AdminCmd::CMD_REPACK, AdminCmd::SUBCMD_ERR):
processRepack_Err(request.admincmd(), response);
@@ -1364,55 +1364,21 @@ void RequestMessage::processRepack_Rm(const cta::admin::AdminCmd &admincmd, cta:
-void RequestMessage::processRepack_Ls(const cta::admin::AdminCmd &admincmd, cta::xrd::Response &response)
+void RequestMessage::processRepack_Ls(const cta::admin::AdminCmd &admincmd, cta::xrd::Response &response, XrdSsiStream* &stream)
{
using namespace cta::admin;
+
+ auto vid = getOptional(OptionString::VID);
- std::stringstream cmdlineOutput;
-
- auto vid = getOptional(OptionString::VID);
-
- std::list<cta::common::dataStructures::RepackInfo> list;
-
- if(!vid) {
- list = m_scheduler.getRepacks();
- } else {
- list.push_back(m_scheduler.getRepack(vid.value()));
- }
+ // Create a XrdSsi stream object to return the results
+ stream = new RepackLsStream(m_scheduler, vid);
- if(!list.empty())
- {
- std::vector<std::vector<std::string>> responseTable;
- std::vector<std::string> header = {
- "vid","file buffer URL","UserProvidedFiles","FilesToRetrieve","BytesToRetrieve","FilesToArchive","BytesToArchive","FailedToRetrieve (files)","FailedToRetrieve (bytes)","FailedToArchive (files)","FailedToArchive (bytes)","LastExpandedFSeq","status"//,"name","host","time"
- };
- if(has_flag(OptionBoolean::SHOW_HEADER)) responseTable.push_back(header);
- for(auto it = list.cbegin(); it != list.cend(); it++)
- {
- std::vector<std::string> currentRow;
- currentRow.push_back(it->vid);
- currentRow.push_back(utils::midEllipsis(it->repackBufferBaseURL,40));//std::to_string(static_cast<unsigned long long>(it->totalFiles)));
- currentRow.push_back(std::to_string(static_cast<unsigned long long>(it->userProvidedFiles)));
- currentRow.push_back(std::to_string(static_cast<unsigned long long>(it->totalFilesToRetrieve)));
- currentRow.push_back(std::to_string(static_cast<unsigned long long>(it->totalBytesToRetrieve)));
- currentRow.push_back(std::to_string(static_cast<unsigned long long>(it->totalFilesToArchive)));//std::to_string(static_cast<unsigned long long>(it->totalSize)));
- currentRow.push_back(std::to_string(static_cast<unsigned long long>(it->totalBytesToArchive)));
- currentRow.push_back(std::to_string(static_cast<unsigned long long>(it->failedFilesToRetrieve)));
- currentRow.push_back(std::to_string(static_cast<unsigned long long>(it->failedBytesToRetrieve)));
- currentRow.push_back(std::to_string(static_cast<unsigned long long>(it->failedFilesToArchive)));
- currentRow.push_back(std::to_string(static_cast<unsigned long long>(it->failedBytesToArchive)));
- currentRow.push_back(std::to_string(static_cast<unsigned long long>(it->lastExpandedFseq)));//std::to_string(static_cast<unsigned long long>(it->filesArchived)));
- currentRow.push_back(common::dataStructures::toString(it->status));
- //currentRow.push_back("-");//it->creationLog.username);
- //currentRow.push_back("-");//it->creationLog.host);
- //currentRow.push_back("-");//timeToString(it->creationLog.time));
- responseTable.push_back(currentRow);
- }
- cmdlineOutput << formatResponse(responseTable);
- }
+ // Should the client display column headers?
+ if(has_flag(OptionBoolean::SHOW_HEADER)) {
+ response.set_show_header(HeaderType::REPACK_LS);
+ }
- response.set_message_txt(cmdlineOutput.str());
- response.set_type(cta::xrd::Response::RSP_SUCCESS);
+ response.set_type(cta::xrd::Response::RSP_SUCCESS);
}
diff --git a/xroot_plugins/XrdSsiCtaRequestMessage.hpp b/xroot_plugins/XrdSsiCtaRequestMessage.hpp
index a0074ec87f90e52de6639ccfcd72a6df1f1585c2..085d77d1047efb59a50d017f3b5d9e7a062b01f8 100644
--- a/xroot_plugins/XrdSsiCtaRequestMessage.hpp
+++ b/xroot_plugins/XrdSsiCtaRequestMessage.hpp
@@ -177,7 +177,7 @@ private:
void processMountPolicy_Ls (const cta::admin::AdminCmd &admincmd, cta::xrd::Response &response);
void processRepack_Add (const cta::admin::AdminCmd &admincmd, cta::xrd::Response &response);
void processRepack_Rm (const cta::admin::AdminCmd &admincmd, cta::xrd::Response &response);
- void processRepack_Ls (const cta::admin::AdminCmd &admincmd, cta::xrd::Response &response);
+ //void processRepack_Ls (const cta::admin::AdminCmd &admincmd, cta::xrd::Response &response);
void processRepack_Err (const cta::admin::AdminCmd &admincmd, cta::xrd::Response &response);
void processRequesterMountRule_Add(const cta::admin::AdminCmd &admincmd, cta::xrd::Response &response);
void processRequesterMountRule_Ch (const cta::admin::AdminCmd &admincmd, cta::xrd::Response &response);
@@ -213,6 +213,7 @@ private:
admincmdstream_t processListPendingRetrieves;
admincmdstream_t processTapePool_Ls;
admincmdstream_t processTape_Ls;
+ admincmdstream_t processRepack_Ls;
/*!
* Log an admin command