[repack] Improved repack ls tabular output

ReleaseNotes.md

@@ -8,6 +8,7 @@ This release contains the CTA software Recommended Access Order (RAO) implemente
 
 - CTA software Recommended Access Order (RAO) implemented for LTO drives
 - Upgraded EOS to 4.8.22-1
+- cta-admin repack ls tabular output improvements
 
 ### Bug fixes
 

cmdline/CtaAdminTextFormatter.cpp

@@ -51,6 +51,37 @@ std::string TextFormatter::timeToStr(const time_t &unixtime) {
   return timeStr;
 }
 
+std::string TextFormatter::secondsToDayHoursMinSec(const uint64_t & seconds){
+  uint64_t secondsSave = seconds;
+  int day = secondsSave / (24 * 3600); 
+  
+  secondsSave = secondsSave % (24 * 3600); 
+  int hour = secondsSave / 3600; 
+
+  secondsSave %= 3600; 
+  int minutes = secondsSave / 60 ; 
+
+  secondsSave %= 60; 
+  std::stringstream ss;
+  if(day){
+    ss << day << "d";
+  }
+  if(hour){
+    ss << hour << "h";
+  }
+  if(minutes){
+    ss << minutes << "m";
+  }
+  ss << secondsSave << "s";
+  return ss.str();
+}
+
+std::string TextFormatter::integerToPercentage(const uint64_t & value){
+  std::stringstream ss;
+  ss << value << "%";
+  return ss.str();
+}
+
 
 std::string TextFormatter::dataSizeToStr(uint64_t value) {
   const std::vector<char> suffix = { 'K', 'M', 'G', 'T', 'P', 'E' };
@@ -581,30 +612,32 @@ void TextFormatter::print(const MountPolicyLsItem &mpls_item) {
 void TextFormatter::printRepackLsHeader() {
   push_back("HEADER");
   push_back(
+    "c.time",
+    "repackTime",
+    "c.user",
     "vid",
     "providedFiles",
-    "totalFilesToRetrieve",
-    "totalBytesToRetrieve",
-    "totalFilesToArchive",
-    "totalBytesToArchive",
-    "retrievedFiles",
-    "archivedFiles",
-    "failedToRetrieveFiles",
+    "totalFiles",
+    "totalBytes",
+    "filesToRetrieve",
+    "filesToArchive",
+    "failed",
     "status"
   );
 }
 
 void TextFormatter::print(const RepackLsItem &rels_item) {
-  push_back(
+ push_back(
+   timeToStr(rels_item.creation_log().time()),
+   secondsToDayHoursMinSec(rels_item.repack_time()),
+   rels_item.creation_log().username(),
    rels_item.vid(),
    rels_item.user_provided_files(),
-   rels_item.total_files_to_retrieve(),
-   dataSizeToStr(rels_item.total_bytes_to_retrieve()),
-   rels_item.total_files_to_archive(),
-   dataSizeToStr(rels_item.total_bytes_to_archive()),
-   rels_item.retrieved_files(),
-   rels_item.archived_files(),
-   rels_item.failed_to_retrieve_files(),
+   rels_item.total_files_to_archive(), //https://gitlab.cern.ch/cta/CTA/-/issues/680#note_3849045
+   dataSizeToStr(rels_item.total_bytes_to_archive()), //https://gitlab.cern.ch/cta/CTA/-/issues/680#note_3849045
+   rels_item.files_left_to_retrieve(), //https://gitlab.cern.ch/cta/CTA/-/issues/680#note_3845829
+   rels_item.files_left_to_archive(), //https://gitlab.cern.ch/cta/CTA/-/issues/680#note_3845829
+   rels_item.total_failed_files(),  //https://gitlab.cern.ch/cta/CTA/-/issues/680#note_3849927
    rels_item.status()
   );
 }

cmdline/CtaAdminTextFormatter.hpp

@@ -141,7 +141,13 @@ private:
 
   //! Convert UNIX time to string
   static std::string timeToStr(const time_t &unixtime);
-
+  
+  //! Convert the number of seconds given in parameter to a string like 1d2h35m6s
+  static std::string secondsToDayHoursMinSec(const uint64_t & seconds);
+  
+  //! Appends the '%' character to the value passed in parameter
+  static std::string integerToPercentage(const uint64_t & value);
+  
   //! Convert data size in bytes to abbreviated string with appropriate size suffix (K/M/G/T/P/E)
   static std::string dataSizeToStr(uint64_t value);
 

common/dataStructures/RepackInfo.hpp

@@ -21,6 +21,8 @@
 #include <string>
 #include <list>
 #include "objectstore/RepackQueueType.hpp"
+#include "EntryLog.hpp"
+#include "common/optional.hpp"
 
 namespace cta {
 namespace common {
@@ -67,10 +69,14 @@ struct RepackInfo {
   uint64_t lastExpandedFseq;
   uint64_t userProvidedFiles;
   uint64_t retrievedFiles;
+  uint64_t retrievedBytes;
   uint64_t archivedFiles;
+  uint64_t archivedBytes;
   bool isExpandFinished;
   bool forceDisabledTape;
   bool noRecall;
+  common::dataStructures::EntryLog creationLog;
+  cta::optional<time_t> repackFinishedTime;
   RepackDestinationInfo::List destinationInfos;
 //  std::string tag;
 //  uint64_t totalFiles;

cta.spec.in

@@ -492,6 +492,7 @@ Currently contains a helper for the client-ar script, which should be installed
 %changelog
 * Fri Sep 25 2020 julien.leduc (at) cern.ch - 3.1-8
 - CTA software Recommended Access Order (RAO) implemented for LTO drives
+- cta-admin repack ls tabular output improvements
 - Upstream EOS 4.8.20-1
 - utils::trimString() now returns an empty string if the string passed in parameter contains only white-space characters
 - cta/CTA#895 [catalogue] RdbmsCatalogue::deleteLogicalLibrary does not delete empty logical library

objectstore/GarbageCollectorTest.cpp

@@ -722,6 +722,7 @@ TEST(ObjectStore, GarbageCollectorRepackRequestPending) {
     repackRequest.setBufferURL("test/buffer/url");
     repackRequest.setOwner(agentReferenceRepackRequest.getAgentAddress());
     repackRequest.setMountPolicy(cta::common::dataStructures::MountPolicy::s_defaultMountPolicyForRepack);
+    repackRequest.setCreationLog(cta::common::dataStructures::EntryLog("test","test",time(nullptr)));
     repackRequest.insert();
   }
   {
@@ -803,6 +804,7 @@ TEST(ObjectStore, GarbageCollectorRepackRequestToExpand) {
     repackRequest.setBufferURL("test/buffer/url");
     repackRequest.setOwner(agentReferenceRepackRequest.getAgentAddress());
     repackRequest.setMountPolicy(cta::common::dataStructures::MountPolicy::s_defaultMountPolicyForRepack);
+    repackRequest.setCreationLog(cta::common::dataStructures::EntryLog("test","test",time(nullptr)));
     repackRequest.insert();
   }
   {
@@ -884,6 +886,7 @@ TEST(ObjectStore, GarbageCollectorRepackRequestRunningExpandNotFinished) {
     repackRequest.setOwner(agentReferenceRepackRequest.getAgentAddress());
     repackRequest.setExpandFinished(false);
     repackRequest.setMountPolicy(cta::common::dataStructures::MountPolicy::s_defaultMountPolicyForRepack);
+    repackRequest.setCreationLog(cta::common::dataStructures::EntryLog("test","test",time(nullptr)));
     repackRequest.insert();
   }
   {
@@ -966,6 +969,7 @@ TEST(ObjectStore, GarbageCollectorRepackRequestRunningExpandFinished) {
     repackRequest.setOwner(agentReferenceRepackRequest.getAgentAddress());
     repackRequest.setExpandFinished(true);
     repackRequest.setMountPolicy(cta::common::dataStructures::MountPolicy::s_defaultMountPolicyForRepack);
+    repackRequest.setCreationLog(cta::common::dataStructures::EntryLog("test","test",time(nullptr)));
     repackRequest.insert();
   }
   cta::log::StringLogger strLogger("dummy", "dummy", cta::log::DEBUG);
@@ -1065,6 +1069,7 @@ TEST(ObjectStore, GarbageCollectorRepackRequestStarting) {
     repackRequest.setOwner(agentReferenceRepackRequest.getAgentAddress());
     repackRequest.setExpandFinished(true);
     repackRequest.setMountPolicy(cta::common::dataStructures::MountPolicy::s_defaultMountPolicyForRepack);
+    repackRequest.setCreationLog(cta::common::dataStructures::EntryLog("test","test",time(nullptr)));
     repackRequest.insert();
   }
   cta::log::StringLogger strLogger("dummy", "dummy", cta::log::DEBUG);

objectstore/RepackRequest.cpp

@@ -144,12 +144,20 @@ common::dataStructures::RepackInfo RepackRequest::getInfo() {
   ret.failedFilesToRetrieve = m_payload.failedtoretrievefiles();
   ret.failedBytesToRetrieve = m_payload.failedtoretrievebytes();
   ret.archivedFiles = m_payload.archivedfiles();
+  ret.archivedBytes = m_payload.archivedbytes();
   ret.retrievedFiles = m_payload.retrievedfiles();
+  ret.retrievedBytes = m_payload.retrievedbytes();
   ret.lastExpandedFseq = m_payload.lastexpandedfseq();
   ret.userProvidedFiles = m_payload.userprovidedfiles();
   ret.isExpandFinished = m_payload.is_expand_finished();
   ret.forceDisabledTape = m_payload.force_disabled_tape();
   ret.noRecall = m_payload.no_recall();
+  EntryLogSerDeser creationLog;
+  creationLog.deserialize(m_payload.creation_log());
+  ret.creationLog = creationLog;
+  if(m_payload.has_repack_finished_time()){
+    ret.repackFinishedTime = m_payload.repack_finished_time();
+  }
   for(auto & rdi: m_payload.destination_infos()){
     RepackInfo::RepackDestinationInfo rdiToInsert;
     rdiToInsert.vid = rdi.vid();
@@ -265,6 +273,19 @@ std::list<common::dataStructures::RepackInfo::RepackDestinationInfo> RepackReque
   return ret;
 }
 
+void RepackRequest::setCreationLog(const common::dataStructures::EntryLog & creationLog){
+  checkPayloadWritable();
+  cta::objectstore::EntryLogSerDeser creationLogToSet(creationLog);
+  creationLogToSet.serialize(*m_payload.mutable_creation_log());
+}
+
+common::dataStructures::EntryLog RepackRequest::getCreationLog() {
+  checkPayloadReadable();
+  cta::objectstore::EntryLogSerDeser ret;
+  ret.deserialize(m_payload.creation_log());
+  return ret;
+}
+
 void RepackRequest::setForceDisabledTape(const bool disabledTape){
   checkPayloadWritable();
   m_payload.set_force_disabled_tape(disabledTape);
@@ -288,7 +309,6 @@ bool RepackRequest::getNoRecall(){
 void RepackRequest::setStatus(){
   checkPayloadWritable();
   checkPayloadReadable();
-  
   if(m_payload.is_expand_started()){
     //The expansion of the Repack Request have started 
     if(m_payload.is_expand_finished()){
@@ -296,9 +316,11 @@ void RepackRequest::setStatus(){
         //We reached the end
         if (m_payload.failedtoretrievefiles() || m_payload.failedtoarchivefiles()) {
           //At least one retrieve or archive has failed
+          m_payload.set_repack_finished_time(time(nullptr));
           setStatus(common::dataStructures::RepackInfo::Status::Failed);
         } else {
           //No Failure, we are status Complete
+          m_payload.set_repack_finished_time(time(nullptr));
           setStatus(common::dataStructures::RepackInfo::Status::Complete);
         }
         return;

objectstore/RepackRequest.hpp

@@ -54,6 +54,8 @@ public:
   void setIsComplete(const bool complete);
   void updateRepackDestinationInfos(const common::dataStructures::ArchiveFile & archiveFile, const std::string & destinationVid);
   std::list<common::dataStructures::RepackInfo::RepackDestinationInfo> getRepackDestinationInfos();
+  void setCreationLog(const common::dataStructures::EntryLog & creationLog);
+  common::dataStructures::EntryLog getCreationLog();
   /**
    * Set the flag disabledTape to allow the mounting of a
    * disabled tape for file retrieval

objectstore/cta.proto

@@ -631,6 +631,8 @@ message RepackRequest {
   required bool no_recall = 11566;
   repeated RepackSubRequestPointer subrequests = 11570;
   repeated RepackDestinationInfo destination_infos = 11571;
+  required EntryLog creation_log = 11572;
+  optional uint64 repack_finished_time = 11573;
 }
 
 message RepackRequestIndexPointer {

scheduler/OStoreDB/OStoreDB.cpp

@@ -1719,6 +1719,7 @@ void OStoreDB::queueRepack(const SchedulerDatabase::QueueRepackRequest & repackR
   rr->setMountPolicy(mountPolicy);
   rr->setForceDisabledTape(forceDisabledTape);
   rr->setNoRecall(repackRequest.m_noRecall);
+  rr->setCreationLog(repackRequest.m_creationLog);
   // Try to reference the object in the index (will fail if there is already a request with this VID.
   try {
     Helpers::registerRepackRequestToIndex(vid, rr->getAddressIfSet(), *m_agentReference, m_objectStore, lc);

scheduler/Scheduler.cpp

@@ -327,13 +327,15 @@ void Scheduler::checkTapeFullBeforeRepack(std::string vid){
 //------------------------------------------------------------------------------
 void Scheduler::queueRepack(const common::dataStructures::SecurityIdentity &cliIdentity, const SchedulerDatabase::QueueRepackRequest & repackRequest, log::LogContext & lc) {
   // Check request sanity
+  SchedulerDatabase::QueueRepackRequest repackRequestToQueue = repackRequest;
+  repackRequestToQueue.m_creationLog = common::dataStructures::EntryLog(cliIdentity.username,cliIdentity.host,::time(nullptr));
   std::string vid = repackRequest.m_vid;
   std::string repackBufferURL = repackRequest.m_repackBufferURL;
   if (vid.empty()) throw exception::UserError("Empty VID name.");
   if (repackBufferURL.empty()) throw exception::UserError("Empty buffer URL.");
   utils::Timer t;
   checkTapeFullBeforeRepack(vid);
-  m_db.queueRepack(repackRequest, lc);
+  m_db.queueRepack(repackRequestToQueue, lc);
   log::TimingList tl;
   tl.insertAndReset("schedulerDbTime", t);
   log::ScopedParamContainer params(lc);
@@ -342,6 +344,9 @@ void Scheduler::queueRepack(const common::dataStructures::SecurityIdentity &cliI
         .add("forceDisabledTape", repackRequest.m_forceDisabledTape)
         .add("mountPolicy", repackRequest.m_mountPolicy.name)
         .add("noRecall", repackRequest.m_noRecall)
+        .add("creationHostName",repackRequestToQueue.m_creationLog.host)
+        .add("creationUserName",repackRequestToQueue.m_creationLog.username)
+        .add("creationTime",repackRequestToQueue.m_creationLog.time)
         .add("bufferURL", repackRequest.m_repackBufferURL);
   tl.addToLog(params);
   lc.log(log::INFO, "In Scheduler::queueRepack(): success.");

scheduler/SchedulerDatabase.hpp

@@ -163,6 +163,7 @@ public:
     common::dataStructures::MountPolicy m_mountPolicy;
     bool m_forceDisabledTape;
     bool m_noRecall;
+    common::dataStructures::EntryLog m_creationLog;
   };
   
   /*============ Archive management: tape server side =======================*/

xroot_plugins/XrdCtaRepackLs.hpp

@@ -35,7 +35,10 @@ namespace cta { namespace xrd {
     XrdSsiStream(XrdSsiStream::isActive),m_scheduler(scheduler), m_vid(vid){
       XrdSsiPb::Log::Msg(XrdSsiPb::Log::DEBUG, LOG_SUFFIX, "RepackLsStream() constructor");
       if(!vid){
-	m_repackList = m_scheduler.getRepacks();
+        m_repackList = m_scheduler.getRepacks();
+        m_repackList.sort([](cta::common::dataStructures::RepackInfo repackInfo1, cta::common::dataStructures::RepackInfo repackInfo2){
+          return repackInfo1.creationLog.time < repackInfo2.creationLog.time;
+        });
       } else {
 	m_repackList.push_back(m_scheduler.getRepack(vid.value()));
       }
@@ -63,21 +66,41 @@ namespace cta { namespace xrd {
 	for(bool is_buffer_full = false; !m_repackList.empty() && !is_buffer_full; m_repackList.pop_front()){
 	  Data record;
 	  auto &repackRequest = m_repackList.front();
+    
+    uint64_t filesLeftToRetrieve = repackRequest.totalFilesToRetrieve - repackRequest.retrievedFiles;
+    uint64_t filesLeftToArchive = repackRequest.totalFilesToArchive - repackRequest.archivedFiles;
+    uint64_t totalFilesToRetrieve = repackRequest.totalFilesToRetrieve;
+    uint64_t totalFilesToArchive = repackRequest.totalFilesToArchive;
+    
 	  auto repackRequestItem = record.mutable_rels_item();
 	  repackRequestItem->set_vid(repackRequest.vid);
 	  repackRequestItem->set_repack_buffer_url(repackRequest.repackBufferBaseURL);
 	  repackRequestItem->set_user_provided_files(repackRequest.userProvidedFiles);
-	  repackRequestItem->set_total_files_to_retrieve(repackRequest.totalFilesToRetrieve);
+	  repackRequestItem->set_total_files_to_retrieve(totalFilesToRetrieve);
 	  repackRequestItem->set_total_bytes_to_retrieve(repackRequest.totalBytesToRetrieve);
-	  repackRequestItem->set_total_files_to_archive(repackRequest.totalFilesToArchive);
+	  repackRequestItem->set_total_files_to_archive(totalFilesToArchive);
 	  repackRequestItem->set_total_bytes_to_archive(repackRequest.totalBytesToArchive);
 	  repackRequestItem->set_retrieved_files(repackRequest.retrievedFiles);
+    repackRequestItem->set_retrieved_bytes(repackRequest.retrievedBytes);
+    repackRequestItem->set_files_left_to_retrieve(filesLeftToRetrieve);
+    repackRequestItem->set_files_left_to_archive(filesLeftToArchive);
 	  repackRequestItem->set_archived_files(repackRequest.archivedFiles);
+    repackRequestItem->set_archived_bytes(repackRequest.archivedBytes);
 	  repackRequestItem->set_failed_to_retrieve_files(repackRequest.failedFilesToRetrieve);
 	  repackRequestItem->set_failed_to_retrieve_bytes(repackRequest.failedBytesToRetrieve);
 	  repackRequestItem->set_failed_to_archive_files(repackRequest.failedFilesToArchive);
 	  repackRequestItem->set_failed_to_archive_bytes(repackRequest.failedBytesToArchive);
+    repackRequestItem->set_total_failed_files(repackRequest.failedFilesToRetrieve + repackRequest.failedFilesToArchive);
 	  repackRequestItem->set_status(toString(repackRequest.status));
+    uint64_t repackTime = time(nullptr) - repackRequest.creationLog.time;
+    if(repackRequest.status == common::dataStructures::RepackInfo::Status::Complete || repackRequest.status == common::dataStructures::RepackInfo::Status::Failed){
+      repackRequestItem->set_repack_finished_time(repackRequest.repackFinishedTime.value());
+      repackTime = repackRequest.repackFinishedTime.value() - repackRequest.creationLog.time;
+    }
+    repackRequestItem->set_repack_time(repackTime);
+    repackRequestItem->mutable_creation_log()->set_username(repackRequest.creationLog.username);
+    repackRequestItem->mutable_creation_log()->set_host(repackRequest.creationLog.host);
+    repackRequestItem->mutable_creation_log()->set_time(repackRequest.creationLog.time);
 	  //Last expanded fSeq is in reality the next FSeq to Expand. So last one is next - 1
 	  repackRequestItem->set_last_expanded_fseq(repackRequest.lastExpandedFseq != 0 ? repackRequest.lastExpandedFseq - 1 : 0);
 	  repackRequestItem->mutable_destination_infos()->Clear();

xroot_plugins/XrdSsiCtaRequestMessage.cpp

@@ -1512,7 +1512,7 @@ void RequestMessage::processRepack_Add(cta::xrd::Response &response)
   bool forceDisabledTape = has_flag(OptionBoolean::DISABLED);
   
   bool noRecall = has_flag(OptionBoolean::NO_RECALL);
-
+  
   // Process each item in the list
   for(auto it = vid_list.begin(); it != vid_list.end(); ++it) {
     SchedulerDatabase::QueueRepackRequest repackRequest(*it,bufferURL,type,mountPolicy,forceDisabledTape, noRecall);

xrootd-ssi-protobuf-interface @ bfcb6f5b

-Subproject commit b74b3edc752da126cf623f96c0a9fa0b10520224
+Subproject commit bfcb6f5ba28baa2a18e737ab16f476e74f44b694