Commit 1a5c0a52 authored by Sebastien Ponce's avatar Sebastien Ponce
Browse files

Added the possibility of specifying the ceph userId in the ceph library and...

Added the possibility of specifying the ceph userId in the ceph library and used it in the CASTOR code
parent 538cc7ff
......@@ -309,6 +309,9 @@
# you can use a list like pool1 pool2 ...
#DiskManager DataPools
# The User Id to be used to access the above pools, e.g. in case of CEPH
#DiskManager PoolUserId castor
# The following definition allow the max and minallowed free space to be defined
# at the disk server level respectively. They apply to both MountPoints and DataPools
#DiskManager FSMaxFreeSpace .05
......
......@@ -416,7 +416,7 @@ BEGIN
AND DiskServer.name = diskServerName
UNION ALL
SELECT DiskCopy.castorFile,
DataPool.name || '/' || DiskCopy.path AS path, DiskCopy.id,
getConfigOption('DiskManager', 'PoolUserId', 'castor') || '@' || DataPool.name || '/' || DiskCopy.path AS path, DiskCopy.id,
DiskCopy.lastAccessTime, DiskCopy.nbCopyAccesses, DiskCopy.gcWeight,
getObjStatusName('DiskCopy', 'gcType', DiskCopy.gcType) AS gcType,
getSvcClassListDP(DiskServer.dataPool) AS svcClassList
......
......@@ -83,7 +83,7 @@ BEGIN
nbCopyAccesses = nbCopyAccesses + 1
WHERE id = varDcId;
IF selectedMountPoint IS NULL THEN
outPath := varDpName || '/' || outPath;
outPath := getConfigOption('DiskManager', 'PoolUserId', 'castor') || '@' || varDpName || '/' || outPath;
ELSE
outPath := selectedMountPoint || outPath;
END IF;
......@@ -196,7 +196,7 @@ BEGIN
RETURNING path, status, diskCopySize
INTO outPath, varDcStatus, varDiskCopySize;
IF selectedMountPoint IS NULL THEN
outPath := varDpName || '/' || outPath;
outPath := getConfigOption('DiskManager', 'PoolUserId', 'castor') || '@' || varDpName || '/' || outPath;
ELSE
outPath := selectedMountPoint || outPath;
END IF;
......
......@@ -35,13 +35,27 @@ librados::IoCtx* castor::gc::getRadosIoCtx(std::string pool) {
return g_ioCtx[pool];
}
libradosstriper::RadosStriper* castor::gc::getRadosStriper(std::string pool) {
libradosstriper::RadosStriper* castor::gc::getRadosStriper(std::string userAtPool) {
std::map<std::string, libradosstriper::RadosStriper*>::iterator it =
g_radosStripers.find(pool);
g_radosStripers.find(userAtPool);
if (it == g_radosStripers.end()) {
// we need to create a new radosStriper
// First find the user id (if any given) in the pool string
// format is [<userid>@]<poolname>
const char* userId = 0;
size_t pos = userAtPool.find('@');
std::string user;
std::string pool;
if (pos != std::string::npos) {
user = userAtPool.substr(0, pos);
userId = user.c_str();
pool = userAtPool.substr(pos+1);
} else {
pool = userAtPool;
}
// Create the Rados object
librados::Rados cluster;
int rc = cluster.init(0);
int rc = cluster.init(userId);
if (rc) return 0;
rc = cluster.conf_read_file(NULL);
if (rc) {
......@@ -54,22 +68,22 @@ libradosstriper::RadosStriper* castor::gc::getRadosStriper(std::string pool) {
cluster.shutdown();
return 0;
}
g_ioCtx[pool] = new librados::IoCtx();
librados::IoCtx *ioctx = g_ioCtx[pool];
g_ioCtx[userAtPool] = new librados::IoCtx();
librados::IoCtx *ioctx = g_ioCtx[userAtPool];
rc = cluster.ioctx_create(pool.c_str(), *ioctx);
if (rc != 0) {
g_ioCtx.erase(pool);
g_ioCtx.erase(userAtPool);
return 0;
}
libradosstriper::RadosStriper *newStriper = new libradosstriper::RadosStriper;
rc = libradosstriper::RadosStriper::striper_create(*ioctx, newStriper);
if (rc != 0) {
delete newStriper;
g_ioCtx.erase(pool);
g_ioCtx.erase(userAtPool);
return 0;
}
it = g_radosStripers.insert(std::pair<std::string, libradosstriper::RadosStriper*>
(pool, newStriper)).first;
(userAtPool, newStriper)).first;
}
return it->second;
}
......@@ -263,10 +263,16 @@ void castor::gc::SynchronizationThread::syncDataPools() {
sleep(m_chunkInterval);
return;
}
const char* userId = getconfent("DiskManager", "PoolUserId", 0);
if (0 == userId) {
userId = "castor";
}
// Loop over the DataPools
std::map<std::string, std::map<u_signed64, std::string> > paths;
for (int dpIt = 0; dpIt < nbDPs; dpIt++) {
g_pool = dps[dpIt];
g_pool = userId;
g_pool += "@";
g_pool += dps[dpIt];
librados::IoCtx* ioCtx = getRadosIoCtx(g_pool);
if (0 == ioCtx) {
castor::dlf::Param params[] =
......
......@@ -95,30 +95,35 @@ class ReporterThread(threading.Thread):
os.mkdir(os.sep.join([mountPoint, '%02d' % i]))
# now handle dataPools if any
if dataPools:
# get numbers from ceph
jsonoutput = subprocess.Popen(['ceph', 'df', '-f', 'json'],
stdout=subprocess.PIPE).communicate()[0]
pyoutput = json.loads(jsonoutput)
usedSpace = dict([(entry['name'], entry['stats']['bytes_used'])
for entry in pyoutput['pools']
if entry['name'] in dataPools])
jsonoutput = subprocess.Popen(['ceph', 'osd', 'dump', '-f', 'json'],
stdout=subprocess.PIPE).communicate()[0]
pyoutput = json.loads(jsonoutput)
quotas = dict([(entry['pool_name'], entry['quota_max_bytes'])
for entry in pyoutput['pools']
if entry['pool_name'] in dataPools])
for dataPool in dataPools:
try:
totalSpace = quotas[dataPool]
if 0 == totalSpace:
raise Exception("Quota missing in datapool %s (set to 0)" % dataPool)
freeSpace = totalSpace - usedSpace[dataPool]
# fill report
reports.append((diskServerName, dataPool, maxFreeSpace, minAllowedFreeSpace,
totalSpace, freeSpace, 0, 0, 0, 0))
except KeyError:
raise Exception("No data found for datapool %s - May be missing quota" % dataPool)
try :
# check which userId to use
userId = self.configuration.getValue('DiskManager', 'PoolUserId', 'castor')
# get numbers from ceph
jsonoutput = subprocess.Popen(['ceph', 'df', '--id', userId, '-f', 'json'],
stdout=subprocess.PIPE).communicate()[0]
pyoutput = json.loads(jsonoutput)
usedSpace = dict([(entry['name'], entry['stats']['bytes_used'])
for entry in pyoutput['pools']
if entry['name'] in dataPools])
jsonoutput = subprocess.Popen(['ceph', 'osd', 'dump', '--id', userId, '-f', 'json'],
stdout=subprocess.PIPE).communicate()[0]
pyoutput = json.loads(jsonoutput)
quotas = dict([(entry['pool_name'], entry['quota_max_bytes'])
for entry in pyoutput['pools']
if entry['pool_name'] in dataPools])
for dataPool in dataPools:
try:
totalSpace = quotas[dataPool]
if 0 == totalSpace:
raise Exception("Quota missing in datapool %s (set to 0)" % dataPool)
freeSpace = totalSpace - usedSpace[dataPool]
# fill report
reports.append((diskServerName, dataPool, maxFreeSpace, minAllowedFreeSpace,
totalSpace, freeSpace, 0, 0, 0, 0))
except KeyError:
raise Exception("No data found for datapool %s - May be missing quota" % dataPool)
except ValueError, e:
raise Exception("Error caught in trying to build report for Datapools : %s" % e)
return tuple(reports)
def run(self):
......
......@@ -56,13 +56,27 @@ static std::pair<std::string, std::string> splitPoolFromObjName(const char* path
spath.substr(slashPos+1));
}
static libradosstriper::RadosStriper* getRadosStriper(std::string pool) {
static libradosstriper::RadosStriper* getRadosStriper(std::string userAtPool) {
std::map<std::string, libradosstriper::RadosStriper*>::iterator it =
g_radosStripers.find(pool);
g_radosStripers.find(userAtPool);
if (it == g_radosStripers.end()) {
// we need to create a new radosStriper
// First find the user id (if any given) in the pool string
// format is [<userid>@]<poolname>
const char* userId = 0;
size_t pos = userAtPool.find('@');
std::string user;
std::string pool;
if (pos != std::string::npos) {
user = userAtPool.substr(0, pos);
userId = user.c_str();
pool = userAtPool.substr(pos+1);
} else {
pool = userAtPool;
}
// Create the Rados object
librados::Rados cluster;
int rc = cluster.init(0);
int rc = cluster.init(userId);
if (rc) return 0;
rc = cluster.conf_read_file(NULL);
if (rc) {
......@@ -88,7 +102,7 @@ static libradosstriper::RadosStriper* getRadosStriper(std::string pool) {
return 0;
}
it = g_radosStripers.insert(std::pair<std::string, libradosstriper::RadosStriper*>
(pool, newStriper)).first;
(userAtPool, newStriper)).first;
}
return it->second;
}
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment