Commit 642397a8 authored by Cedric Caffy's avatar Cedric Caffy
Browse files

[repack] Improved repack ls tabular output

parent a2d6e8c6
......@@ -8,6 +8,7 @@ This release contains the CTA software Recommended Access Order (RAO) implemente
- CTA software Recommended Access Order (RAO) implemented for LTO drives
- Upgraded EOS to 4.8.22-1
- cta-admin repack ls tabular output improvements
### Bug fixes
......
......@@ -51,6 +51,37 @@ std::string TextFormatter::timeToStr(const time_t &unixtime) {
return timeStr;
}
std::string TextFormatter::secondsToDayHoursMinSec(const uint64_t & seconds){
uint64_t secondsSave = seconds;
int day = secondsSave / (24 * 3600);
secondsSave = secondsSave % (24 * 3600);
int hour = secondsSave / 3600;
secondsSave %= 3600;
int minutes = secondsSave / 60 ;
secondsSave %= 60;
std::stringstream ss;
if(day){
ss << day << "d";
}
if(hour){
ss << hour << "h";
}
if(minutes){
ss << minutes << "m";
}
ss << secondsSave << "s";
return ss.str();
}
std::string TextFormatter::integerToPercentage(const uint64_t & value){
std::stringstream ss;
ss << value << "%";
return ss.str();
}
std::string TextFormatter::dataSizeToStr(uint64_t value) {
const std::vector<char> suffix = { 'K', 'M', 'G', 'T', 'P', 'E' };
......@@ -581,30 +612,32 @@ void TextFormatter::print(const MountPolicyLsItem &mpls_item) {
void TextFormatter::printRepackLsHeader() {
push_back("HEADER");
push_back(
"c.time",
"repackTime",
"c.user",
"vid",
"providedFiles",
"totalFilesToRetrieve",
"totalBytesToRetrieve",
"totalFilesToArchive",
"totalBytesToArchive",
"retrievedFiles",
"archivedFiles",
"failedToRetrieveFiles",
"totalFiles",
"totalBytes",
"filesToRetrieve",
"filesToArchive",
"failed",
"status"
);
}
void TextFormatter::print(const RepackLsItem &rels_item) {
push_back(
push_back(
timeToStr(rels_item.creation_log().time()),
secondsToDayHoursMinSec(rels_item.repack_time()),
rels_item.creation_log().username(),
rels_item.vid(),
rels_item.user_provided_files(),
rels_item.total_files_to_retrieve(),
dataSizeToStr(rels_item.total_bytes_to_retrieve()),
rels_item.total_files_to_archive(),
dataSizeToStr(rels_item.total_bytes_to_archive()),
rels_item.retrieved_files(),
rels_item.archived_files(),
rels_item.failed_to_retrieve_files(),
rels_item.total_files_to_archive(), //https://gitlab.cern.ch/cta/CTA/-/issues/680#note_3849045
dataSizeToStr(rels_item.total_bytes_to_archive()), //https://gitlab.cern.ch/cta/CTA/-/issues/680#note_3849045
rels_item.files_left_to_retrieve(), //https://gitlab.cern.ch/cta/CTA/-/issues/680#note_3845829
rels_item.files_left_to_archive(), //https://gitlab.cern.ch/cta/CTA/-/issues/680#note_3845829
rels_item.total_failed_files(), //https://gitlab.cern.ch/cta/CTA/-/issues/680#note_3849927
rels_item.status()
);
}
......
......@@ -141,7 +141,13 @@ private:
//! Convert UNIX time to string
static std::string timeToStr(const time_t &unixtime);
//! Convert the number of seconds given in parameter to a string like 1d2h35m6s
static std::string secondsToDayHoursMinSec(const uint64_t & seconds);
//! Appends the '%' character to the value passed in parameter
static std::string integerToPercentage(const uint64_t & value);
//! Convert data size in bytes to abbreviated string with appropriate size suffix (K/M/G/T/P/E)
static std::string dataSizeToStr(uint64_t value);
......
......@@ -21,6 +21,8 @@
#include <string>
#include <list>
#include "objectstore/RepackQueueType.hpp"
#include "EntryLog.hpp"
#include "common/optional.hpp"
namespace cta {
namespace common {
......@@ -67,10 +69,14 @@ struct RepackInfo {
uint64_t lastExpandedFseq;
uint64_t userProvidedFiles;
uint64_t retrievedFiles;
uint64_t retrievedBytes;
uint64_t archivedFiles;
uint64_t archivedBytes;
bool isExpandFinished;
bool forceDisabledTape;
bool noRecall;
common::dataStructures::EntryLog creationLog;
cta::optional<time_t> repackFinishedTime;
RepackDestinationInfo::List destinationInfos;
// std::string tag;
// uint64_t totalFiles;
......
......@@ -492,6 +492,7 @@ Currently contains a helper for the client-ar script, which should be installed
%changelog
* Fri Sep 25 2020 julien.leduc (at) cern.ch - 3.1-8
- CTA software Recommended Access Order (RAO) implemented for LTO drives
- cta-admin repack ls tabular output improvements
- Upstream EOS 4.8.20-1
- utils::trimString() now returns an empty string if the string passed in parameter contains only white-space characters
- cta/CTA#895 [catalogue] RdbmsCatalogue::deleteLogicalLibrary does not delete empty logical library
......
......@@ -722,6 +722,7 @@ TEST(ObjectStore, GarbageCollectorRepackRequestPending) {
repackRequest.setBufferURL("test/buffer/url");
repackRequest.setOwner(agentReferenceRepackRequest.getAgentAddress());
repackRequest.setMountPolicy(cta::common::dataStructures::MountPolicy::s_defaultMountPolicyForRepack);
repackRequest.setCreationLog(cta::common::dataStructures::EntryLog("test","test",time(nullptr)));
repackRequest.insert();
}
{
......@@ -803,6 +804,7 @@ TEST(ObjectStore, GarbageCollectorRepackRequestToExpand) {
repackRequest.setBufferURL("test/buffer/url");
repackRequest.setOwner(agentReferenceRepackRequest.getAgentAddress());
repackRequest.setMountPolicy(cta::common::dataStructures::MountPolicy::s_defaultMountPolicyForRepack);
repackRequest.setCreationLog(cta::common::dataStructures::EntryLog("test","test",time(nullptr)));
repackRequest.insert();
}
{
......@@ -884,6 +886,7 @@ TEST(ObjectStore, GarbageCollectorRepackRequestRunningExpandNotFinished) {
repackRequest.setOwner(agentReferenceRepackRequest.getAgentAddress());
repackRequest.setExpandFinished(false);
repackRequest.setMountPolicy(cta::common::dataStructures::MountPolicy::s_defaultMountPolicyForRepack);
repackRequest.setCreationLog(cta::common::dataStructures::EntryLog("test","test",time(nullptr)));
repackRequest.insert();
}
{
......@@ -966,6 +969,7 @@ TEST(ObjectStore, GarbageCollectorRepackRequestRunningExpandFinished) {
repackRequest.setOwner(agentReferenceRepackRequest.getAgentAddress());
repackRequest.setExpandFinished(true);
repackRequest.setMountPolicy(cta::common::dataStructures::MountPolicy::s_defaultMountPolicyForRepack);
repackRequest.setCreationLog(cta::common::dataStructures::EntryLog("test","test",time(nullptr)));
repackRequest.insert();
}
cta::log::StringLogger strLogger("dummy", "dummy", cta::log::DEBUG);
......@@ -1065,6 +1069,7 @@ TEST(ObjectStore, GarbageCollectorRepackRequestStarting) {
repackRequest.setOwner(agentReferenceRepackRequest.getAgentAddress());
repackRequest.setExpandFinished(true);
repackRequest.setMountPolicy(cta::common::dataStructures::MountPolicy::s_defaultMountPolicyForRepack);
repackRequest.setCreationLog(cta::common::dataStructures::EntryLog("test","test",time(nullptr)));
repackRequest.insert();
}
cta::log::StringLogger strLogger("dummy", "dummy", cta::log::DEBUG);
......
......@@ -144,12 +144,20 @@ common::dataStructures::RepackInfo RepackRequest::getInfo() {
ret.failedFilesToRetrieve = m_payload.failedtoretrievefiles();
ret.failedBytesToRetrieve = m_payload.failedtoretrievebytes();
ret.archivedFiles = m_payload.archivedfiles();
ret.archivedBytes = m_payload.archivedbytes();
ret.retrievedFiles = m_payload.retrievedfiles();
ret.retrievedBytes = m_payload.retrievedbytes();
ret.lastExpandedFseq = m_payload.lastexpandedfseq();
ret.userProvidedFiles = m_payload.userprovidedfiles();
ret.isExpandFinished = m_payload.is_expand_finished();
ret.forceDisabledTape = m_payload.force_disabled_tape();
ret.noRecall = m_payload.no_recall();
EntryLogSerDeser creationLog;
creationLog.deserialize(m_payload.creation_log());
ret.creationLog = creationLog;
if(m_payload.has_repack_finished_time()){
ret.repackFinishedTime = m_payload.repack_finished_time();
}
for(auto & rdi: m_payload.destination_infos()){
RepackInfo::RepackDestinationInfo rdiToInsert;
rdiToInsert.vid = rdi.vid();
......@@ -265,6 +273,19 @@ std::list<common::dataStructures::RepackInfo::RepackDestinationInfo> RepackReque
return ret;
}
void RepackRequest::setCreationLog(const common::dataStructures::EntryLog & creationLog){
checkPayloadWritable();
cta::objectstore::EntryLogSerDeser creationLogToSet(creationLog);
creationLogToSet.serialize(*m_payload.mutable_creation_log());
}
common::dataStructures::EntryLog RepackRequest::getCreationLog() {
checkPayloadReadable();
cta::objectstore::EntryLogSerDeser ret;
ret.deserialize(m_payload.creation_log());
return ret;
}
void RepackRequest::setForceDisabledTape(const bool disabledTape){
checkPayloadWritable();
m_payload.set_force_disabled_tape(disabledTape);
......@@ -288,7 +309,6 @@ bool RepackRequest::getNoRecall(){
void RepackRequest::setStatus(){
checkPayloadWritable();
checkPayloadReadable();
if(m_payload.is_expand_started()){
//The expansion of the Repack Request have started
if(m_payload.is_expand_finished()){
......@@ -296,9 +316,11 @@ void RepackRequest::setStatus(){
//We reached the end
if (m_payload.failedtoretrievefiles() || m_payload.failedtoarchivefiles()) {
//At least one retrieve or archive has failed
m_payload.set_repack_finished_time(time(nullptr));
setStatus(common::dataStructures::RepackInfo::Status::Failed);
} else {
//No Failure, we are status Complete
m_payload.set_repack_finished_time(time(nullptr));
setStatus(common::dataStructures::RepackInfo::Status::Complete);
}
return;
......
......@@ -54,6 +54,8 @@ public:
void setIsComplete(const bool complete);
void updateRepackDestinationInfos(const common::dataStructures::ArchiveFile & archiveFile, const std::string & destinationVid);
std::list<common::dataStructures::RepackInfo::RepackDestinationInfo> getRepackDestinationInfos();
void setCreationLog(const common::dataStructures::EntryLog & creationLog);
common::dataStructures::EntryLog getCreationLog();
/**
* Set the flag disabledTape to allow the mounting of a
* disabled tape for file retrieval
......
......@@ -631,6 +631,8 @@ message RepackRequest {
required bool no_recall = 11566;
repeated RepackSubRequestPointer subrequests = 11570;
repeated RepackDestinationInfo destination_infos = 11571;
required EntryLog creation_log = 11572;
optional uint64 repack_finished_time = 11573;
}
message RepackRequestIndexPointer {
......
......@@ -1719,6 +1719,7 @@ void OStoreDB::queueRepack(const SchedulerDatabase::QueueRepackRequest & repackR
rr->setMountPolicy(mountPolicy);
rr->setForceDisabledTape(forceDisabledTape);
rr->setNoRecall(repackRequest.m_noRecall);
rr->setCreationLog(repackRequest.m_creationLog);
// Try to reference the object in the index (will fail if there is already a request with this VID.
try {
Helpers::registerRepackRequestToIndex(vid, rr->getAddressIfSet(), *m_agentReference, m_objectStore, lc);
......
......@@ -327,13 +327,15 @@ void Scheduler::checkTapeFullBeforeRepack(std::string vid){
//------------------------------------------------------------------------------
void Scheduler::queueRepack(const common::dataStructures::SecurityIdentity &cliIdentity, const SchedulerDatabase::QueueRepackRequest & repackRequest, log::LogContext & lc) {
// Check request sanity
SchedulerDatabase::QueueRepackRequest repackRequestToQueue = repackRequest;
repackRequestToQueue.m_creationLog = common::dataStructures::EntryLog(cliIdentity.username,cliIdentity.host,::time(nullptr));
std::string vid = repackRequest.m_vid;
std::string repackBufferURL = repackRequest.m_repackBufferURL;
if (vid.empty()) throw exception::UserError("Empty VID name.");
if (repackBufferURL.empty()) throw exception::UserError("Empty buffer URL.");
utils::Timer t;
checkTapeFullBeforeRepack(vid);
m_db.queueRepack(repackRequest, lc);
m_db.queueRepack(repackRequestToQueue, lc);
log::TimingList tl;
tl.insertAndReset("schedulerDbTime", t);
log::ScopedParamContainer params(lc);
......@@ -342,6 +344,9 @@ void Scheduler::queueRepack(const common::dataStructures::SecurityIdentity &cliI
.add("forceDisabledTape", repackRequest.m_forceDisabledTape)
.add("mountPolicy", repackRequest.m_mountPolicy.name)
.add("noRecall", repackRequest.m_noRecall)
.add("creationHostName",repackRequestToQueue.m_creationLog.host)
.add("creationUserName",repackRequestToQueue.m_creationLog.username)
.add("creationTime",repackRequestToQueue.m_creationLog.time)
.add("bufferURL", repackRequest.m_repackBufferURL);
tl.addToLog(params);
lc.log(log::INFO, "In Scheduler::queueRepack(): success.");
......
......@@ -163,6 +163,7 @@ public:
common::dataStructures::MountPolicy m_mountPolicy;
bool m_forceDisabledTape;
bool m_noRecall;
common::dataStructures::EntryLog m_creationLog;
};
/*============ Archive management: tape server side =======================*/
......
......@@ -35,7 +35,10 @@ namespace cta { namespace xrd {
XrdSsiStream(XrdSsiStream::isActive),m_scheduler(scheduler), m_vid(vid){
XrdSsiPb::Log::Msg(XrdSsiPb::Log::DEBUG, LOG_SUFFIX, "RepackLsStream() constructor");
if(!vid){
m_repackList = m_scheduler.getRepacks();
m_repackList = m_scheduler.getRepacks();
m_repackList.sort([](cta::common::dataStructures::RepackInfo repackInfo1, cta::common::dataStructures::RepackInfo repackInfo2){
return repackInfo1.creationLog.time < repackInfo2.creationLog.time;
});
} else {
m_repackList.push_back(m_scheduler.getRepack(vid.value()));
}
......@@ -63,21 +66,41 @@ namespace cta { namespace xrd {
for(bool is_buffer_full = false; !m_repackList.empty() && !is_buffer_full; m_repackList.pop_front()){
Data record;
auto &repackRequest = m_repackList.front();
uint64_t filesLeftToRetrieve = repackRequest.totalFilesToRetrieve - repackRequest.retrievedFiles;
uint64_t filesLeftToArchive = repackRequest.totalFilesToArchive - repackRequest.archivedFiles;
uint64_t totalFilesToRetrieve = repackRequest.totalFilesToRetrieve;
uint64_t totalFilesToArchive = repackRequest.totalFilesToArchive;
auto repackRequestItem = record.mutable_rels_item();
repackRequestItem->set_vid(repackRequest.vid);
repackRequestItem->set_repack_buffer_url(repackRequest.repackBufferBaseURL);
repackRequestItem->set_user_provided_files(repackRequest.userProvidedFiles);
repackRequestItem->set_total_files_to_retrieve(repackRequest.totalFilesToRetrieve);
repackRequestItem->set_total_files_to_retrieve(totalFilesToRetrieve);
repackRequestItem->set_total_bytes_to_retrieve(repackRequest.totalBytesToRetrieve);
repackRequestItem->set_total_files_to_archive(repackRequest.totalFilesToArchive);
repackRequestItem->set_total_files_to_archive(totalFilesToArchive);
repackRequestItem->set_total_bytes_to_archive(repackRequest.totalBytesToArchive);
repackRequestItem->set_retrieved_files(repackRequest.retrievedFiles);
repackRequestItem->set_retrieved_bytes(repackRequest.retrievedBytes);
repackRequestItem->set_files_left_to_retrieve(filesLeftToRetrieve);
repackRequestItem->set_files_left_to_archive(filesLeftToArchive);
repackRequestItem->set_archived_files(repackRequest.archivedFiles);
repackRequestItem->set_archived_bytes(repackRequest.archivedBytes);
repackRequestItem->set_failed_to_retrieve_files(repackRequest.failedFilesToRetrieve);
repackRequestItem->set_failed_to_retrieve_bytes(repackRequest.failedBytesToRetrieve);
repackRequestItem->set_failed_to_archive_files(repackRequest.failedFilesToArchive);
repackRequestItem->set_failed_to_archive_bytes(repackRequest.failedBytesToArchive);
repackRequestItem->set_total_failed_files(repackRequest.failedFilesToRetrieve + repackRequest.failedFilesToArchive);
repackRequestItem->set_status(toString(repackRequest.status));
uint64_t repackTime = time(nullptr) - repackRequest.creationLog.time;
if(repackRequest.status == common::dataStructures::RepackInfo::Status::Complete || repackRequest.status == common::dataStructures::RepackInfo::Status::Failed){
repackRequestItem->set_repack_finished_time(repackRequest.repackFinishedTime.value());
repackTime = repackRequest.repackFinishedTime.value() - repackRequest.creationLog.time;
}
repackRequestItem->set_repack_time(repackTime);
repackRequestItem->mutable_creation_log()->set_username(repackRequest.creationLog.username);
repackRequestItem->mutable_creation_log()->set_host(repackRequest.creationLog.host);
repackRequestItem->mutable_creation_log()->set_time(repackRequest.creationLog.time);
//Last expanded fSeq is in reality the next FSeq to Expand. So last one is next - 1
repackRequestItem->set_last_expanded_fseq(repackRequest.lastExpandedFseq != 0 ? repackRequest.lastExpandedFseq - 1 : 0);
repackRequestItem->mutable_destination_infos()->Clear();
......
......@@ -1512,7 +1512,7 @@ void RequestMessage::processRepack_Add(cta::xrd::Response &response)
bool forceDisabledTape = has_flag(OptionBoolean::DISABLED);
bool noRecall = has_flag(OptionBoolean::NO_RECALL);
// Process each item in the list
for(auto it = vid_list.begin(); it != vid_list.end(); ++it) {
SchedulerDatabase::QueueRepackRequest repackRequest(*it,bufferURL,type,mountPolicy,forceDisabledTape, noRecall);
......
Subproject commit b74b3edc752da126cf623f96c0a9fa0b10520224
Subproject commit bfcb6f5ba28baa2a18e737ab16f476e74f44b694
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment