diff --git a/continuousintegration/orchestration/tests/repack_generate_report.sh b/continuousintegration/orchestration/tests/repack_generate_report.sh index 6669460b2081792ca91e6809f6c25b35b43fd198..8182ca5c79ec8233d1cf0cacb832149d8c1eaa99 100755 --- a/continuousintegration/orchestration/tests/repack_generate_report.sh +++ b/continuousintegration/orchestration/tests/repack_generate_report.sh @@ -70,7 +70,7 @@ echo "OK" echo "5. Report of the repacked tape" echo -NB_NON_REPACKED_FILES=$(jq '[.[]] | length' ${NOT_REPACKED_JSON_PATH}) +NB_NON_REPACKED_FILES=$(jq '[.[]] | length' ${NOT_REPACKED_JSON_PATH} || 0) echo "Number of non repacked files : ${NB_NON_REPACKED_FILES}" if [ ${NB_NON_REPACKED_FILES} -ne 0 ] then @@ -78,7 +78,7 @@ then { echo -e $header; jq -r '.[] | [.af.archiveId,.tf.fSeq, .af.size] | @tsv' ${NOT_REPACKED_JSON_PATH}; } | column -t fi; echo -NB_SELF_REPACKED_FILES=$(jq '[.[]] | length' ${SELF_REPACKED_JSON_PATH}) +NB_SELF_REPACKED_FILES=$(jq '[.[]] | length' ${SELF_REPACKED_JSON_PATH} || 0) echo "Number of self-repacked files : ${NB_SELF_REPACKED_FILES}" if [ ${NB_SELF_REPACKED_FILES} -ne 0 ] then @@ -86,7 +86,7 @@ then { echo -e $header; jq -r '.[] | [.af.archiveId, .tf.fSeq, .af.size] | @tsv' ${SELF_REPACKED_JSON_PATH}; } | column -t fi; echo -NB_REPACKED_FILES=$(jq '[.[]] | length' ${REPACKED_JSON_PATH}) +NB_REPACKED_FILES=$(jq '[.[]] | length' ${REPACKED_JSON_PATH} || 0) echo "Number of repacked files : ${NB_REPACKED_FILES}" if [ ${NB_REPACKED_FILES} -ne 0 ] then diff --git a/continuousintegration/orchestration/tests/repack_systemtest.sh b/continuousintegration/orchestration/tests/repack_systemtest.sh index 58818b2edad52a5e28dac03b98d56782d8c045ea..f6a911190cd1fa28ef017e4afbfa4f43639d2b68 100755 --- a/continuousintegration/orchestration/tests/repack_systemtest.sh +++ b/continuousintegration/orchestration/tests/repack_systemtest.sh @@ -50,7 +50,7 @@ while getopts "v:e:b:t:" o; do REPACK_BUFFER_BASEDIR=${OPTARG} ;; t) - WAIT_FOR_REPACK_TIMEOUT={$OPTARG} + WAIT_FOR_REPACK_TIMEOUT=${OPTARG} ;; *) usage @@ -104,4 +104,4 @@ if test 1 = `admin_cta repack ls --vid ${VID_TO_REPACK} | grep -E "Failed" | wc exit 1 fi -./root/repack_generate_report.sh -v ${VID_TO_REPACK} \ No newline at end of file +exec /root/repack_generate_report.sh -v ${VID_TO_REPACK} \ No newline at end of file diff --git a/scheduler/OStoreDB/OStoreDB.cpp b/scheduler/OStoreDB/OStoreDB.cpp index 825b7a67c217feed73c74ad011c0863d33a233f4..b378b9e31816409b9d17331da43f5b220bbbef7b 100644 --- a/scheduler/OStoreDB/OStoreDB.cpp +++ b/scheduler/OStoreDB/OStoreDB.cpp @@ -1591,6 +1591,26 @@ std::unique_ptr<SchedulerDatabase::RepackReportBatch> OStoreDB::getNextRepackRep return nullptr; } +//------------------------------------------------------------------------------ +// OStoreDB::getRepackReportBatches() +//------------------------------------------------------------------------------ +std::list<std::unique_ptr<SchedulerDatabase::RepackReportBatch>> OStoreDB::getRepackReportBatches(log::LogContext &lc){ + std::list<std::unique_ptr<SchedulerDatabase::RepackReportBatch>> ret; + try{ + ret.push_back(std::move(getNextSuccessfulRetrieveRepackReportBatch(lc))); + } catch (const NoRepackReportBatchFound &){} + try{ + ret.push_back(std::move(getNextFailedRetrieveRepackReportBatch(lc))); + } catch (const NoRepackReportBatchFound &){} + try{ + ret.push_back(std::move(getNextSuccessfulArchiveRepackReportBatch(lc))); + } catch (const NoRepackReportBatchFound &){} + try{ + ret.push_back(std::move(getNextFailedArchiveRepackReportBatch(lc))); + } catch (const NoRepackReportBatchFound &){} + return ret; +} + //------------------------------------------------------------------------------ // OStoreDB::getNextSuccessfulRetrieveRepackReportBatch() //------------------------------------------------------------------------------ diff --git a/scheduler/OStoreDB/OStoreDB.hpp b/scheduler/OStoreDB/OStoreDB.hpp index a5daab47a625e58bf97a5e412ea38a1073e1d1c4..97c4ad316a05e11bbb030445af97bd28ad731fd9 100644 --- a/scheduler/OStoreDB/OStoreDB.hpp +++ b/scheduler/OStoreDB/OStoreDB.hpp @@ -500,6 +500,9 @@ public: }; std::unique_ptr<SchedulerDatabase::RepackReportBatch> getNextRepackReportBatch(log::LogContext& lc) override; + + std::list<std::unique_ptr<SchedulerDatabase::RepackReportBatch>> getRepackReportBatches(log::LogContext &lc) override; + private: CTA_GENERATE_EXCEPTION_CLASS(NoRepackReportBatchFound); const size_t c_repackReportBatchSize = 500; diff --git a/scheduler/OStoreDB/OStoreDBFactory.hpp b/scheduler/OStoreDB/OStoreDBFactory.hpp index 1e9694ebdcdea1616838228219186b3a00de90c4..b578aedce0b2731c402c5541a803ffffbb9b8912 100644 --- a/scheduler/OStoreDB/OStoreDBFactory.hpp +++ b/scheduler/OStoreDB/OStoreDBFactory.hpp @@ -157,6 +157,10 @@ public: std::unique_ptr<RepackReportBatch> getNextRepackReportBatch(log::LogContext& lc) override { return m_OStoreDB.getNextRepackReportBatch(lc); } + + std::list<std::unique_ptr<SchedulerDatabase::RepackReportBatch>> getRepackReportBatches(log::LogContext &lc) override { + return m_OStoreDB.getRepackReportBatches(lc); + } JobsFailedSummary getRetrieveJobsFailedSummary(log::LogContext &lc) override { return m_OStoreDB.getRetrieveJobsFailedSummary(lc); diff --git a/scheduler/RepackRequestManager.cpp b/scheduler/RepackRequestManager.cpp index cf5fbf7b6adc9ef497334c9eafbe8a5384c3a218..09c69a09f9f4d02820ee9e4b08bbb566c30b9b3c 100644 --- a/scheduler/RepackRequestManager.cpp +++ b/scheduler/RepackRequestManager.cpp @@ -48,8 +48,10 @@ void RepackRequestManager::runOnePass(log::LogContext& lc) { } { - // Do one round of repack subrequest reporting (heavy lifting is done internally). - m_scheduler.getNextRepackReportBatch(lc).report(lc); + // Do all round of repack subrequest reporting (heavy lifting is done internally). + for(auto& reportBatch: m_scheduler.getRepackReportBatches(lc)){ + reportBatch.report(lc); + } } } diff --git a/scheduler/Scheduler.cpp b/scheduler/Scheduler.cpp index 6d6670763e1886d6395321c1db83e82839020916..16f0d76829485c41d2dbda2b76bcd94f366fb4f5 100644 --- a/scheduler/Scheduler.cpp +++ b/scheduler/Scheduler.cpp @@ -586,6 +586,16 @@ Scheduler::RepackReportBatch Scheduler::getNextRepackReportBatch(log::LogContext return ret; } +std::list<Scheduler::RepackReportBatch> Scheduler::getRepackReportBatches(log::LogContext &lc){ + std::list<Scheduler::RepackReportBatch> ret; + for(auto& reportBatch: m_db.getRepackReportBatches(lc)){ + Scheduler::RepackReportBatch report; + report.m_DbBatch.reset(reportBatch.release()); + ret.push_back(std::move(report)); + } + return ret; +} + //------------------------------------------------------------------------------ // Scheduler::RepackReportBatch::report //------------------------------------------------------------------------------ diff --git a/scheduler/Scheduler.hpp b/scheduler/Scheduler.hpp index d0e2aa2985b6bd7078d081d91a10966abd082262..d19820ed86e91aa9ab21d2ac6473805ab869f386 100644 --- a/scheduler/Scheduler.hpp +++ b/scheduler/Scheduler.hpp @@ -347,6 +347,8 @@ public: bool empty() { return nullptr == m_DbBatch; } }; RepackReportBatch getNextRepackReportBatch(log::LogContext & lc); + std::list<Scheduler::RepackReportBatch> getRepackReportBatches(log::LogContext &lc); + /*======================= Failed archive jobs support ======================*/ SchedulerDatabase::JobsFailedSummary getArchiveJobsFailedSummary(log::LogContext &lc); diff --git a/scheduler/SchedulerDatabase.hpp b/scheduler/SchedulerDatabase.hpp index ee1de1f9ba1c223e66c5a0995f4c47411d70b087..c58874553c05aaf371df6380ba44b45f0a96bb4c 100644 --- a/scheduler/SchedulerDatabase.hpp +++ b/scheduler/SchedulerDatabase.hpp @@ -492,6 +492,13 @@ public: */ virtual std::unique_ptr<RepackReportBatch> getNextRepackReportBatch(log::LogContext & lc) = 0; + /** + * Return all batches of subrequests from the database to be reported to repack. + * @param lc log context + * @return the list of all batches to be reported + */ + virtual std::list<std::unique_ptr<RepackReportBatch>> getRepackReportBatches(log::LogContext &lc) = 0; + /** * Set a batch of jobs as reported (modeled on ArchiveMount::setJobBatchSuccessful(). * @param jobsBatch