From f785d7d69a3ece54369d6cd511c06e9b366e3907 Mon Sep 17 00:00:00 2001
From: mvelosob <miguel.veloso.barros@cern.ch>
Date: Wed, 6 Apr 2022 15:22:17 +0200
Subject: [PATCH] Add support for configurable scheduler stack sizes (#1168)
---
ReleaseNotes.md | 2 +-
common/threading/Thread.cpp | 12 +++++++++++-
common/threading/Thread.hpp | 6 +++++-
scheduler/OStoreDB/OStoreDB.cpp | 4 ++--
scheduler/OStoreDB/OStoreDB.hpp | 5 ++++-
xroot_plugins/XrdSsiCtaServiceProvider.cpp | 8 +++++++-
xroot_plugins/cta-frontend-xrootd.conf | 1 +
7 files changed, 31 insertions(+), 7 deletions(-)
diff --git a/ReleaseNotes.md b/ReleaseNotes.md
index dfc372f41a..113903d9c0 100644
--- a/ReleaseNotes.md
+++ b/ReleaseNotes.md
@@ -9,7 +9,7 @@ Please consult the [database upgrade documentation](https://eoscta.docs.cern.ch/
### Features
- cta/CTA#1163 - cta-admin now prefixes the drivename with a '!' if the respective logical library is disabled
-
+- cta/CTA#1168 - Add configuration option for scheduler stack size to /etc/cta/cta-frontend-xrootd.conf
### Bug fixes
- cta/CTA#1156 - failed to instantiate the RAO algorithm
diff --git a/common/threading/Thread.cpp b/common/threading/Thread.cpp
index bf7b7f35cc..dd449464ab 100644
--- a/common/threading/Thread.cpp
+++ b/common/threading/Thread.cpp
@@ -31,8 +31,18 @@ namespace threading {
//------------------------------------------------------------------------------
void Thread::start()
{
+ pthread_attr_t attr;
cta::exception::Errnum::throwOnReturnedErrno(
- pthread_create(&m_thread, NULL, pthread_runner, this),
+ pthread_attr_init(&attr),
+ "Error from pthread_attr_init in cta::threading::Thread::start()");
+
+ if (m_stackSize) {
+ cta::exception::Errnum::throwOnReturnedErrno(
+ pthread_attr_setstacksize(&attr, m_stackSize.value()),
+ "Error from pthread_attr_setstacksize in cta::threading::Thread::start()");
+ }
+ cta::exception::Errnum::throwOnReturnedErrno(
+ pthread_create(&m_thread, &attr, pthread_runner, this),
"Error from pthread_create in cta::threading::Thread::start()");
m_started = true;
}
diff --git a/common/threading/Thread.hpp b/common/threading/Thread.hpp
index 8fad1714e4..b7c55f0c06 100644
--- a/common/threading/Thread.hpp
+++ b/common/threading/Thread.hpp
@@ -20,6 +20,7 @@
#include "common/exception/Errnum.hpp"
#include "common/exception/Exception.hpp"
#include "common/threading/Mutex.hpp"
+#include "common/optional.hpp"
#include <pthread.h>
#include <semaphore.h>
@@ -40,6 +41,8 @@ CTA_GENERATE_EXCEPTION_CLASS(UncaughtExceptionInThread);
class Thread {
public:
Thread(): m_hadException(false), m_what(""), m_started(false) {}
+ explicit Thread(cta::optional<size_t> stackSize): m_hadException(false), m_what(""), m_started(false), m_stackSize(stackSize) {}
+
virtual ~Thread () {}
void start() ;
void wait() ;
@@ -52,7 +55,8 @@ private:
std::string m_what;
std::string m_type;
static void * pthread_runner (void * arg);
- bool m_started;
+ bool m_started;
+ cta::optional<size_t> m_stackSize;
};
} // namespace threading
diff --git a/scheduler/OStoreDB/OStoreDB.cpp b/scheduler/OStoreDB/OStoreDB.cpp
index 157aa0f8fe..943fb32e09 100644
--- a/scheduler/OStoreDB/OStoreDB.cpp
+++ b/scheduler/OStoreDB/OStoreDB.cpp
@@ -106,7 +106,7 @@ void OStoreDB::waitSubthreadsComplete() {
//------------------------------------------------------------------------------
// OStoreDB::setThreadNumber()
//------------------------------------------------------------------------------
-void OStoreDB::setThreadNumber(uint64_t threadNumber) {
+void OStoreDB::setThreadNumber(uint64_t threadNumber, const cta::optional<size_t> &stackSize) {
// Clear all threads.
for (__attribute__((unused)) auto &t: m_enqueueingWorkerThreads) m_enqueueingTasksQueue.push(nullptr);
for (auto &t: m_enqueueingWorkerThreads) {
@@ -117,7 +117,7 @@ void OStoreDB::setThreadNumber(uint64_t threadNumber) {
m_enqueueingWorkerThreads.clear();
// Create the new ones.
for (size_t i=0; i<threadNumber; i++) {
- m_enqueueingWorkerThreads.emplace_back(new EnqueueingWorkerThread(m_enqueueingTasksQueue));
+ m_enqueueingWorkerThreads.emplace_back(new EnqueueingWorkerThread(m_enqueueingTasksQueue, stackSize));
m_enqueueingWorkerThreads.back()->start();
}
}
diff --git a/scheduler/OStoreDB/OStoreDB.hpp b/scheduler/OStoreDB/OStoreDB.hpp
index 5bbd464f3d..589063e5a0 100644
--- a/scheduler/OStoreDB/OStoreDB.hpp
+++ b/scheduler/OStoreDB/OStoreDB.hpp
@@ -76,6 +76,9 @@ class OStoreDB: public SchedulerDatabase {
public:
explicit EnqueueingWorkerThread(cta::threading::BlockingQueue<EnqueueingTask*> & etq)
: m_enqueueingTasksQueue(etq) {}
+ EnqueueingWorkerThread(cta::threading::BlockingQueue<EnqueueingTask*> & etq, cta::optional<size_t> stackSize)
+ : cta::threading::Thread(stackSize), m_enqueueingTasksQueue(etq) {}
+
void start() { cta::threading::Thread::start(); }
void wait() { cta::threading::Thread::wait(); }
private:
@@ -90,7 +93,7 @@ class OStoreDB: public SchedulerDatabase {
public:
void waitSubthreadsComplete() override;
- void setThreadNumber(uint64_t threadNumber);
+ void setThreadNumber(uint64_t threadNumber, const cta::optional<size_t> &stackSize = cta::nullopt);
void setBottomHalfQueueSize(uint64_t tasksNumber);
/*============ Basic IO check: validate object store access ===============*/
void ping() override;
diff --git a/xroot_plugins/XrdSsiCtaServiceProvider.cpp b/xroot_plugins/XrdSsiCtaServiceProvider.cpp
index 2c7ec46ff5..fedbf70e01 100644
--- a/xroot_plugins/XrdSsiCtaServiceProvider.cpp
+++ b/xroot_plugins/XrdSsiCtaServiceProvider.cpp
@@ -194,9 +194,15 @@ void XrdSsiCtaServiceProvider::ExceptionThrowingInit(XrdSsiLogger *logP, XrdSsiC
m_scheddb_init = cta::make_unique<SchedulerDBInit_t>("Frontend", db_conn.second, *m_log);
m_scheddb = m_scheddb_init->getSchedDB(*m_catalogue, *m_log);
+ cta::optional<size_t> schedulerThreadStackOpt;
+ const auto schedulerThreadStackSize = config.getOptionValueInt("ca.schedulerdb.threadstacksize_mb");
+ if (schedulerThreadStackSize.first) {
+ schedulerThreadStackOpt = schedulerThreadStackSize.second * 1024 * 1024;
+ }
+
auto threadPoolSize = config.getOptionValueInt("cta.schedulerdb.numberofthreads");
if (threadPoolSize.first) {
- m_scheddb->setThreadNumber(threadPoolSize.second);
+ m_scheddb->setThreadNumber(threadPoolSize.second, schedulerThreadStackOpt);
}
m_scheddb->setBottomHalfQueueSize(25000);
diff --git a/xroot_plugins/cta-frontend-xrootd.conf b/xroot_plugins/cta-frontend-xrootd.conf
index a0f14fc16e..ea7b9e7d43 100644
--- a/xroot_plugins/cta-frontend-xrootd.conf
+++ b/xroot_plugins/cta-frontend-xrootd.conf
@@ -7,6 +7,7 @@
# CTA Scheduler DB options
cta.schedulerdb.numberofthreads 500
+cta.schedulerdb.threadstacksize_mb 1
# CTA Catalogue options
cta.catalogue.numberofconnections 10
--
GitLab