Commit 57cdc2a1 authored by Sebastien Ponce's avatar Sebastien Ponce
Browse files

Added -R/--recursive option to modifydiskserver and fixed many small issues in the server side code

parent 69575bac
......@@ -27,10 +27,9 @@
'''This module allows to modify the properties of a set of diskservers
(e.g. state or diskPool) from a remote location such as the diskserver itself.'''
import sys
import castor_tools
def modifyDiskServers(dbconn, targets, state, mountPoints, diskPool):
def modifyDiskServers(dbconn, targets, state, mountPoints, diskPool, isRecursive):
'''modifies the properties of a set of diskservers'''
stcur = dbconn.cursor()
# check target diskpool if any
......@@ -39,14 +38,9 @@ def modifyDiskServers(dbconn, targets, state, mountPoints, diskPool):
stcur.execute(sqlStatement, diskPool=diskPool)
rows = stcur.fetchall()
if not rows:
print 'DiskPool %d does not exist. Giving up' % diskPool
try:
castor_tools.disconnectDB(dbconn)
except Exception:
pass
sys.exit(1)
raise ValueError('DiskPool %s does not exist. Giving up' % diskPool)
diskPoolId = rows[0][0]
# check diskpools
# check target diskpools
sqlStatement = 'SELECT id, name FROM DiskPool WHERE name = :dpname'
diskPools = set([])
diskPoolIds = []
......@@ -56,7 +50,7 @@ def modifyDiskServers(dbconn, targets, state, mountPoints, diskPool):
if row:
diskPoolIds.append(row[0])
diskPools.add(row[1])
# check diskservers
# check target diskservers
sqlStatement = 'SELECT id, name FROM DiskServer WHERE name = :dsname'
diskServers = set([])
diskServerIds = []
......@@ -68,11 +62,41 @@ def modifyDiskServers(dbconn, targets, state, mountPoints, diskPool):
diskServers.add(row[1])
unknownTargets = set(t for t in targets) - diskPools - diskServers
if not diskPools and not diskServers:
returnMsg = 'None of the provided diskpools/diskservers could be found. Giving up'
return returnMsg
raise ValueError('None of the provided diskpools/diskservers could be found. Giving up')
# go for the update
if mountPoints:
# check MountPoints
returnMsg = []
mountPointIds = []
if mountPoints == None:
# check DiskServers
if diskPools:
sqlStatement = '''SELECT UNIQUE DiskServer.id FROM DiskServer, FileSystem
WHERE DiskServer.id = FileSystem.diskServer
AND FileSystem.diskPool IN (''' + "'" + \
"', '".join([str(x) for x in diskPoolIds]) + "')"
stcur.execute(sqlStatement)
rows = stcur.fetchall()
diskServerIds.extend([row[0] for row in rows])
if not diskServerIds:
raise ValueError('No diskserver matching your request')
# update diskServers for status
if state != None:
sqlStatement = '''UPDATE DiskServer SET status = :status
WHERE id IN (''' + ', '.join([str(x) for x in diskServerIds]) + ')'
stcur.execute(sqlStatement, status=state)
# update fileSystems for diskPool
if diskPool:
sqlStatement = '''UPDATE FileSystem SET diskPool = :diskPoolId
WHERE diskServer IN (''' + ', '.join([str(x) for x in diskServerIds]) + ')'
stcur.execute(sqlStatement, diskPoolId=diskPoolId)
returnMsg.append('Diskserver(s) modified successfully')
# in case we are running in recursive mode, list mountPoints
if isRecursive:
sqlStatement = 'SELECT id FROM FileSystem WHERE diskserver IN (' + \
', '.join([str(x) for x in diskServerIds]) + ')'
stcur.execute(sqlStatement)
mountPointIds = [row[0] for row in stcur.fetchall()]
else:
# MountPoints were given by the user, check them
sqlStatement = '''SELECT id, mountPoint FROM FileSystem
WHERE mountPoint = :mountPoint'''
if targets:
......@@ -84,7 +108,6 @@ def modifyDiskServers(dbconn, targets, state, mountPoints, diskPool):
sqlStatement += ' OR '
sqlStatement += 'diskServer IN (' + ', '.join([str(x) for x in diskServerIds]) + ')'
sqlStatement += ')'
mountPointIds = []
existingMountPoints = set([])
for mountPoint in mountPoints:
stcur.execute(sqlStatement, mountPoint=mountPoint)
......@@ -93,57 +116,29 @@ def modifyDiskServers(dbconn, targets, state, mountPoints, diskPool):
mountPointIds.append(row[0])
existingMountPoints.add(row[1])
unknownMountPoints = set(m for m in mountPoints) - existingMountPoints
# update mountPoints
varDict = {}
sqlStatement = 'UPDATE FileSystem SET '
setClause = ''
whereClause = ' WHERE id IN (' + ', '.join([str(x) for x in mountPointIds]) + ')'
if state != None:
setClause = 'status = :status'
varDict['status'] = state
# and set FileSystems' DiskPool if needed
if diskPool:
if setClause:
setClause += ','
setClause += ' diskPool = :diskPool'
varDict['diskPool'] = diskPoolId
stcur.execute(sqlStatement + setClause + whereClause, varDict)
# commit update and tell user
dbconn.commit()
returnMsg = 'Filesystem(s) modified successfully'
# mention unknown mountPoints
if mountPoints and unknownMountPoints:
returnMsg += '\nWARNING : the following mountPoints could not be found : ' + ', '.join(unknownMountPoints)
else:
# check DiskServers
if diskPools:
sqlStatement = '''SELECT UNIQUE DiskServer.id FROM DiskServer, FileSystem
WHERE DiskServer.id = FileSystem.diskServer
AND FileSystem.diskPool IN (''' + "'" + \
"', '".join([str(x) for x in diskPoolIds]) + "')"
stcur.execute(sqlStatement)
rows = stcur.fetchall()
diskServerIds.extend([row[0] for row in rows])
if not diskServerIds:
returnMsg = 'No diskserver matching your request'
# update diskServers for status
if diskServerIds:
if state != None:
varDict = {}
sqlStatement = '''UPDATE DiskServer SET status = :status
WHERE id IN (''' + ', '.join([str(x) for x in diskServerIds]) + ')'
varDict['status'] = state
stcur.execute(sqlStatement, varDict)
# update fileSystems for diskPool
if diskPool:
varDict = {}
sqlStatement = '''UPDATE FileSystem SET diskPool = :diskPoolId
WHERE diskServer IN (''' + ', '.join([str(x) for x in diskServerIds]) + ')'
stcur.execute(sqlStatement, diskPoolId=diskPoolId)
# commit update and tell user
dbconn.commit()
returnMsg = 'Diskserver(s) modified successfully'
sqlStatement = 'UPDATE FileSystem SET diskPool = :diskPool WHERE id IN (' + \
', '.join([str(x) for x in mountPointIds]) + ')'
stcur.execute(sqlStatement, diskPool=diskPoolId)
returnMsg.append('Filesystem(s)\' diskpools modified successfully')
# Now status of mountpoints
# they may have been given by the user or created by the handling of the recursive option
if mountPointIds:
# update mountPoints
sqlStatement = 'UPDATE FileSystem SET status = :status WHERE id IN (' + \
', '.join([str(x) for x in mountPointIds]) + ')'
stcur.execute(sqlStatement, status=state)
returnMsg.append('Filesystem(s)\' status modified successfully')
# commit updates
dbconn.commit()
# mention unknown mountPoints
if mountPoints and unknownMountPoints:
returnMsg.append('WARNING : the following mountPoints could not be found : ' + \
', '.join(unknownMountPoints))
# mention unknown targets
if unknownTargets:
returnMsg += '\nWARNING : the following diskpools/diskservers do not exist : ' + ', '.join(unknownTargets)
returnMsg.append('WARNING : the following diskpools/diskservers do not exist : ' + \
', '.join(unknownTargets))
# return collected messages. If any exception, this is forwarded to the caller
return returnMsg
return '\n'.join(returnMsg)
......@@ -521,13 +521,13 @@ class TransferManagerService(rpyc.Service):
Returns the status of the diskserver and all its fileSystems'''
reportmanager.handleStateReport(report)
def exposed_modifyDiskServers(self, targets, state, mountPoints, diskPool):
def exposed_modifyDiskServers(self, targets, state, mountPoints, diskPool, isRecursive):
'''called by the modifydiskserver CLI to modify a set of diskservers. Returns a user-friendly message .'''
dlf.write(msgs.INVOKINGMODIFYDISKSERVERS, Targets=str(targets), State=state, MountPoints=str(mountPoints))
try:
try:
self.__class__.acquireDbConnection()
return modifydiskservers.modifyDiskServers(self.__class__.dbConnection(), targets, state, mountPoints, diskPool)
return modifydiskservers.modifyDiskServers(self.__class__.dbConnection(), targets, state, mountPoints, diskPool, isRecursive)
except Exception, e:
# "Exception caught while modifying diskserver(s), giving up" message
dlf.writeerr(msgs.MODIFYDISKSERVERSEXCEPTION, Message=str(e), Type=str(e.__class__), Targets=str(targets), State=state, MountPoints=str(mountPoints), DiskPool=diskPool)
......
......@@ -33,7 +33,7 @@ def usage(exitCode):
'''prints usage'''
print 'Usage : ' + sys.argv[0] + ' [-h|--help] ' + \
'-s|--state <state>|-d|--diskpool <diskpool> ' + \
'[-m|--mountpoints <mountpoint>[:...]] <diskpool>|<diskServerName> [...]]'
'[-m|--mountpoints <mountpoint>[:...]] [-r|--recursive] <diskpool>|<diskServerName> [...]]'
print ' where state can be "Production", "Draining", "Disabled", "Readonly"'
sys.exit(exitCode)
......@@ -42,8 +42,8 @@ states = ['Production', 'Draining', 'Disabled', 'Readonly']
# first parse the options
try:
options, args = getopt.getopt(sys.argv[1:], 'hvs:m:d:',
['help', 'verbose', 'state=', 'mountpoints=', 'diskpool='])
options, args = getopt.getopt(sys.argv[1:], 'hvs:m:d:R',
['help', 'verbose', 'state=', 'mountpoints=', 'diskpool=', 'recursive'])
except Exception, e:
print e
usage(1)
......@@ -51,6 +51,7 @@ verbose = False
state = None
mountPoints = None
diskPool = None
isRecursive = False
for f, v in options:
if f == '-h' or f == '--help':
usage(0)
......@@ -65,6 +66,8 @@ for f, v in options:
mountPoints = tuple(v.split(':'))
elif f == '-d' or f == '--diskpool':
diskPool = v
elif f == '-R' or f == '--recursive':
isRecursive = True
else:
print "unknown option : " + f
usage(1)
......@@ -76,6 +79,9 @@ if not state and not diskPool:
if len(args) == 0:
print 'Nothing to do ! Please give a few diskpools/diskservers'
usage(1)
if mountPoints and isRecursive:
print '--recursive and --mountPoints options are incompatible. Giving up'
usage(1)
try:
if state:
......@@ -83,8 +89,11 @@ try:
conf = castor_tools.castorConf()
rpycconn = rpyc.connect(conf.getValue('DiskManager', 'ServerHosts').split()[0],
conf.getValue('TransferManager', 'Port', 15011, int))
returnMsg = rpycconn.root.modifyDiskServers(args, state, mountPoints, diskPool)
returnMsg = rpycconn.root.modifyDiskServers(args, state, mountPoints, diskPool, isRecursive)
print returnMsg
except ValueError, e:
print e
sys.exit(-1)
except Exception, e:
print 'Caught exception : (' + str(e.__class__) + ') : ' + str(e)
if verbose:
......
......@@ -12,9 +12,12 @@ modifydiskserver \- modifies existing disk servers in the stager catalog
.BI -d|--diskpool
\<diskPoolName>
[
.BI -m|--mountPoints
.BI -m|--mountpoints
\<mountPoint>[:...]
]
[
.BI -R|--recursive
]
<diskPoolName>|<diskServerName>
[...]
......@@ -36,10 +39,14 @@ Allows to change the diskservers/filesystems state to the given value.
.BI \-d,\-\-diskpool\ <diskPoolName>
Allows to move the filesystems given, or all filesystem of the given diskservers to a new diskPool.
.TP
.BI \-m,\-\-mountPoints\ <mountPoint>[:...]
.BI \-m,\-\-mountpoints\ <mountPoint>[:...]
If given, applies the changes to this set of mountpoints rather than to the diskservers.
.TP
.BI -R,\ --recursive
If given, then changes will apply to all mountPoint of the concerned DiskServers, on top of the diskServers themselves.
Incompatible with the --mountpoints option
Note that when -m,--mountPoints is used with -s,--state, the statuses changed will only be the ones of the filesystems. DiskServer states won't be touched.
Note that when -m,--mountpoints is used with -s,--state, the statuses changed will only be the ones of the filesystems. DiskServer states won't be touched.
Symmetrically, when it is not given, the fileSystems' states are untouched.
The targeted mountPoints are all mountPoints with the given name accross the given diskServers.
......@@ -53,7 +60,7 @@ When diskPools are given, all diskServers of the diskPool are considered.
.nf
.ft CW
#printdiskserver
# printdiskserver
DSNAME NBMOUNTPOINTS STATUS ONLINE FREE SIZE NBRD NBWR NBMIGR NBREC DISKPOOL ID
-------------------------------------------------------------------------------------------------------------------
lxc2dev1d1.cern.ch 10 DISKSERVER_PRODUCTION YES 259GiB 275GiB 0 0 0 0 default 6
......@@ -61,13 +68,13 @@ lxc2dev1d2.cern.ch 10 DISKSERVER_PRODUCTION YES 259GiB 275GiB 0
-------------------------------------------------------------------------------------------------------------------
TOTAL NB : 2 20 519GiB 551GiB 0 0 0 0 default,extra
#modifydiskserver -s Disabled lxc2dev1d1.cern.ch
# modifydiskserver -s Disabled lxc2dev1d1.cern.ch
Diskserver(s) modified successfully
#modifydiskserver -s Draining extra
# modifydiskserver -s Draining extra
Filesystem(s) modified successfully
#printdiskserver
# printdiskserver
DSNAME NBMOUNTPOINTS STATUS ONLINE FREE SIZE NBRD NBWR NBMIGR NBREC DISKPOOL ID
-----------------------------------------------------------------------------------------------------------------
lxc2dev1d1.cern.ch 10 DISKSERVER_DISABLED YES 259GiB 275GiB 0 0 0 0 default 6
......@@ -75,13 +82,13 @@ lxc2dev1d2.cern.ch 10 DISKSERVER_DRAINING YES 259GiB 275GiB 0
-----------------------------------------------------------------------------------------------------------------
TOTAL NB : 2 20 519GiB 551GiB 0 0 0 0 default,extra
#modifydiskserver -s Disabled -m /srv/castor/01/:/srv/castor/02/ extra
# modifydiskserver -s Disabled -m /srv/castor/01/:/srv/castor/02/ extra
Filesystem(s) modified successfully
#modifydiskserver -s Production extra
# modifydiskserver -s Production extra
Diskserver(s) modified successfully
#printdiskserver -f
# printdiskserver -f
DSNAME MOUNTPOINT STATUS ONLINE FREE MINFREE MAXFREE SIZE NBRD NBWR NBMIGR NBREC DISKPOOL ID
-------------------------------------------------------------------------------------------------------------------------------------
lxc2dev1d1.cern.ch DISKSERVER_DISABLED YES 259GiB 5% 15% 275GiB 0 0 0 0 default 6
......@@ -109,13 +116,13 @@ lxc2dev1d2.cern.ch DISKSERVER_DRAINING YES 259GiB 5%
-------------------------------------------------------------------------------------------------------------------------------------
TOTAL NB : 2 TOTAL NB : 20 519GiB 5% 15% 551GiB 0 0 0 0 default,extra
#modifydiskserver -d extra default
# modifydiskserver -d extra default
Diskserver(s) modified successfully
#modifydiskserver -d default -m /srv/castor/01/:/srv/castor/02/ lxc2dev1d1.cern.ch
# modifydiskserver -d default -m /srv/castor/01/:/srv/castor/02/ lxc2dev1d1.cern.ch
Filesystem(s) modified successfully
#printdiskserver -f
# printdiskserver -f
DSNAME MOUNTPOINT STATUS ONLINE FREE MINFREE MAXFREE SIZE NBRD NBWR NBMIGR NBREC DISKPOOL ID
-------------------------------------------------------------------------------------------------------------------------------------
lxc2dev1d1.cern.ch DISKSERVER_DISABLED YES 259GiB 5% 15% 275GiB 0 0 0 0 default,extra 6
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment