From 73a3f7bedea17a406856092a15c024b96f051803 Mon Sep 17 00:00:00 2001
From: Cedric CAFFY <cedric.caffy@cern.ch>
Date: Mon, 3 Jun 2019 14:08:51 +0200
Subject: [PATCH] Repack Request Manager now reports every repack subrequests'
report batches
---
.../tests/repack_generate_report.sh | 6 +++---
.../orchestration/tests/repack_systemtest.sh | 4 ++--
scheduler/OStoreDB/OStoreDB.cpp | 20 +++++++++++++++++++
scheduler/OStoreDB/OStoreDB.hpp | 3 +++
scheduler/OStoreDB/OStoreDBFactory.hpp | 4 ++++
scheduler/RepackRequestManager.cpp | 6 ++++--
scheduler/Scheduler.cpp | 10 ++++++++++
scheduler/Scheduler.hpp | 2 ++
scheduler/SchedulerDatabase.hpp | 7 +++++++
9 files changed, 55 insertions(+), 7 deletions(-)
diff --git a/continuousintegration/orchestration/tests/repack_generate_report.sh b/continuousintegration/orchestration/tests/repack_generate_report.sh
index 6669460b20..8182ca5c79 100755
--- a/continuousintegration/orchestration/tests/repack_generate_report.sh
+++ b/continuousintegration/orchestration/tests/repack_generate_report.sh
@@ -70,7 +70,7 @@ echo "OK"
echo "5. Report of the repacked tape"
echo
-NB_NON_REPACKED_FILES=$(jq '[.[]] | length' ${NOT_REPACKED_JSON_PATH})
+NB_NON_REPACKED_FILES=$(jq '[.[]] | length' ${NOT_REPACKED_JSON_PATH} || 0)
echo "Number of non repacked files : ${NB_NON_REPACKED_FILES}"
if [ ${NB_NON_REPACKED_FILES} -ne 0 ]
then
@@ -78,7 +78,7 @@ then
{ echo -e $header; jq -r '.[] | [.af.archiveId,.tf.fSeq, .af.size] | @tsv' ${NOT_REPACKED_JSON_PATH}; } | column -t
fi;
echo
-NB_SELF_REPACKED_FILES=$(jq '[.[]] | length' ${SELF_REPACKED_JSON_PATH})
+NB_SELF_REPACKED_FILES=$(jq '[.[]] | length' ${SELF_REPACKED_JSON_PATH} || 0)
echo "Number of self-repacked files : ${NB_SELF_REPACKED_FILES}"
if [ ${NB_SELF_REPACKED_FILES} -ne 0 ]
then
@@ -86,7 +86,7 @@ then
{ echo -e $header; jq -r '.[] | [.af.archiveId, .tf.fSeq, .af.size] | @tsv' ${SELF_REPACKED_JSON_PATH}; } | column -t
fi;
echo
-NB_REPACKED_FILES=$(jq '[.[]] | length' ${REPACKED_JSON_PATH})
+NB_REPACKED_FILES=$(jq '[.[]] | length' ${REPACKED_JSON_PATH} || 0)
echo "Number of repacked files : ${NB_REPACKED_FILES}"
if [ ${NB_REPACKED_FILES} -ne 0 ]
then
diff --git a/continuousintegration/orchestration/tests/repack_systemtest.sh b/continuousintegration/orchestration/tests/repack_systemtest.sh
index 58818b2eda..f6a911190c 100755
--- a/continuousintegration/orchestration/tests/repack_systemtest.sh
+++ b/continuousintegration/orchestration/tests/repack_systemtest.sh
@@ -50,7 +50,7 @@ while getopts "v:e:b:t:" o; do
REPACK_BUFFER_BASEDIR=${OPTARG}
;;
t)
- WAIT_FOR_REPACK_TIMEOUT={$OPTARG}
+ WAIT_FOR_REPACK_TIMEOUT=${OPTARG}
;;
*)
usage
@@ -104,4 +104,4 @@ if test 1 = `admin_cta repack ls --vid ${VID_TO_REPACK} | grep -E "Failed" | wc
exit 1
fi
-./root/repack_generate_report.sh -v ${VID_TO_REPACK}
\ No newline at end of file
+exec /root/repack_generate_report.sh -v ${VID_TO_REPACK}
\ No newline at end of file
diff --git a/scheduler/OStoreDB/OStoreDB.cpp b/scheduler/OStoreDB/OStoreDB.cpp
index 825b7a67c2..b378b9e318 100644
--- a/scheduler/OStoreDB/OStoreDB.cpp
+++ b/scheduler/OStoreDB/OStoreDB.cpp
@@ -1591,6 +1591,26 @@ std::unique_ptr<SchedulerDatabase::RepackReportBatch> OStoreDB::getNextRepackRep
return nullptr;
}
+//------------------------------------------------------------------------------
+// OStoreDB::getRepackReportBatches()
+//------------------------------------------------------------------------------
+std::list<std::unique_ptr<SchedulerDatabase::RepackReportBatch>> OStoreDB::getRepackReportBatches(log::LogContext &lc){
+ std::list<std::unique_ptr<SchedulerDatabase::RepackReportBatch>> ret;
+ try{
+ ret.push_back(std::move(getNextSuccessfulRetrieveRepackReportBatch(lc)));
+ } catch (const NoRepackReportBatchFound &){}
+ try{
+ ret.push_back(std::move(getNextFailedRetrieveRepackReportBatch(lc)));
+ } catch (const NoRepackReportBatchFound &){}
+ try{
+ ret.push_back(std::move(getNextSuccessfulArchiveRepackReportBatch(lc)));
+ } catch (const NoRepackReportBatchFound &){}
+ try{
+ ret.push_back(std::move(getNextFailedArchiveRepackReportBatch(lc)));
+ } catch (const NoRepackReportBatchFound &){}
+ return ret;
+}
+
//------------------------------------------------------------------------------
// OStoreDB::getNextSuccessfulRetrieveRepackReportBatch()
//------------------------------------------------------------------------------
diff --git a/scheduler/OStoreDB/OStoreDB.hpp b/scheduler/OStoreDB/OStoreDB.hpp
index a5daab47a6..97c4ad316a 100644
--- a/scheduler/OStoreDB/OStoreDB.hpp
+++ b/scheduler/OStoreDB/OStoreDB.hpp
@@ -500,6 +500,9 @@ public:
};
std::unique_ptr<SchedulerDatabase::RepackReportBatch> getNextRepackReportBatch(log::LogContext& lc) override;
+
+ std::list<std::unique_ptr<SchedulerDatabase::RepackReportBatch>> getRepackReportBatches(log::LogContext &lc) override;
+
private:
CTA_GENERATE_EXCEPTION_CLASS(NoRepackReportBatchFound);
const size_t c_repackReportBatchSize = 500;
diff --git a/scheduler/OStoreDB/OStoreDBFactory.hpp b/scheduler/OStoreDB/OStoreDBFactory.hpp
index 1e9694ebdc..b578aedce0 100644
--- a/scheduler/OStoreDB/OStoreDBFactory.hpp
+++ b/scheduler/OStoreDB/OStoreDBFactory.hpp
@@ -157,6 +157,10 @@ public:
std::unique_ptr<RepackReportBatch> getNextRepackReportBatch(log::LogContext& lc) override {
return m_OStoreDB.getNextRepackReportBatch(lc);
}
+
+ std::list<std::unique_ptr<SchedulerDatabase::RepackReportBatch>> getRepackReportBatches(log::LogContext &lc) override {
+ return m_OStoreDB.getRepackReportBatches(lc);
+ }
JobsFailedSummary getRetrieveJobsFailedSummary(log::LogContext &lc) override {
return m_OStoreDB.getRetrieveJobsFailedSummary(lc);
diff --git a/scheduler/RepackRequestManager.cpp b/scheduler/RepackRequestManager.cpp
index cf5fbf7b6a..09c69a09f9 100644
--- a/scheduler/RepackRequestManager.cpp
+++ b/scheduler/RepackRequestManager.cpp
@@ -48,8 +48,10 @@ void RepackRequestManager::runOnePass(log::LogContext& lc) {
}
{
- // Do one round of repack subrequest reporting (heavy lifting is done internally).
- m_scheduler.getNextRepackReportBatch(lc).report(lc);
+ // Do all round of repack subrequest reporting (heavy lifting is done internally).
+ for(auto& reportBatch: m_scheduler.getRepackReportBatches(lc)){
+ reportBatch.report(lc);
+ }
}
}
diff --git a/scheduler/Scheduler.cpp b/scheduler/Scheduler.cpp
index 6d6670763e..16f0d76829 100644
--- a/scheduler/Scheduler.cpp
+++ b/scheduler/Scheduler.cpp
@@ -586,6 +586,16 @@ Scheduler::RepackReportBatch Scheduler::getNextRepackReportBatch(log::LogContext
return ret;
}
+std::list<Scheduler::RepackReportBatch> Scheduler::getRepackReportBatches(log::LogContext &lc){
+ std::list<Scheduler::RepackReportBatch> ret;
+ for(auto& reportBatch: m_db.getRepackReportBatches(lc)){
+ Scheduler::RepackReportBatch report;
+ report.m_DbBatch.reset(reportBatch.release());
+ ret.push_back(std::move(report));
+ }
+ return ret;
+}
+
//------------------------------------------------------------------------------
// Scheduler::RepackReportBatch::report
//------------------------------------------------------------------------------
diff --git a/scheduler/Scheduler.hpp b/scheduler/Scheduler.hpp
index d0e2aa2985..d19820ed86 100644
--- a/scheduler/Scheduler.hpp
+++ b/scheduler/Scheduler.hpp
@@ -347,6 +347,8 @@ public:
bool empty() { return nullptr == m_DbBatch; }
};
RepackReportBatch getNextRepackReportBatch(log::LogContext & lc);
+ std::list<Scheduler::RepackReportBatch> getRepackReportBatches(log::LogContext &lc);
+
/*======================= Failed archive jobs support ======================*/
SchedulerDatabase::JobsFailedSummary getArchiveJobsFailedSummary(log::LogContext &lc);
diff --git a/scheduler/SchedulerDatabase.hpp b/scheduler/SchedulerDatabase.hpp
index ee1de1f9ba..c58874553c 100644
--- a/scheduler/SchedulerDatabase.hpp
+++ b/scheduler/SchedulerDatabase.hpp
@@ -492,6 +492,13 @@ public:
*/
virtual std::unique_ptr<RepackReportBatch> getNextRepackReportBatch(log::LogContext & lc) = 0;
+ /**
+ * Return all batches of subrequests from the database to be reported to repack.
+ * @param lc log context
+ * @return the list of all batches to be reported
+ */
+ virtual std::list<std::unique_ptr<RepackReportBatch>> getRepackReportBatches(log::LogContext &lc) = 0;
+
/**
* Set a batch of jobs as reported (modeled on ArchiveMount::setJobBatchSuccessful().
* @param jobsBatch
--
GitLab