Commit 1b4c285d authored by Giuseppe Lo Presti's avatar Giuseppe Lo Presti
Browse files

[CASTOR migration] Added a script for completing the process (#479).

Also, packaged the python scripts in the same cta-migration RPM.
The scripts are still WIP.
parent 25651a93
......@@ -314,6 +314,8 @@ directory metadata into the EOS namespace.
%attr(0755,root,root) %{_bindir}/json-pretty-print.sh
%attr(0755,root,root) %{_bindir}/exporttapepool.sh
%attr(0755,root,root) %{_bindir}/undoexporttapepool.sh
%attr(0755,root,root) %{_bindir}/tapepool_castor_to_cta.py
%attr(0755,root,root) %{_bindir}/complete_tapepool_export.py
%attr(0644,root,root) %config(noreplace) %{_sysconfdir}/cta/castor-migration.conf.example
%package -n cta-rmcd
......
......@@ -16,4 +16,6 @@
install(FILES ${CMAKE_SOURCE_DIR}/migration/castor/exporttapepool.sh
${CMAKE_SOURCE_DIR}/migration/castor/undoexporttapepool.sh
${CMAKE_SOURCE_DIR}/migration/castor/tapepool_castor_to_cta.py
${CMAKE_SOURCE_DIR}/migration/castor/complete_tapepool_export.py
DESTINATION usr/bin)
#!/usr/bin/python
#/******************************************************************************
# * complete_tapepool_export.py
# *
# * This file is part of the Castor/CTA project.
# * See http://cern.ch/castor and http://cern.ch/eoscta
# * Copyright (C) 2019 CERN
# *
# * This program is free software; you can redistribute it and/or
# * modify it under the terms of the GNU General Public License
# * as published by the Free Software Foundation; either version 2
# * of the License, or (at your option) any later version.
# * This program is distributed in the hope that it will be useful,
# * but WITHOUT ANY WARRANTY; without even the implied warranty of
# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# * GNU General Public License for more details.
# * You should have received a copy of the GNU General Public License
# * along with this program; if not, write to the Free Software
# * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# *
# * @author Castor Dev team, castor-dev@cern.ch
# *****************************************************************************/
'''command line tool to complete the export of a tapepool from CASTOR to CTA'''
import sys
import getopt
from time import sleep
from datetime import datetime
from threading import Thread
import castor_tools
# usage function
def usage(exitcode):
'''prints usage'''
print 'Usage : ' + sys.argv[0] + ' [-h|--help] -t|--tapepool <tapepool> [-d|--dryrun]'
sys.exit(exitcode)
def async_complete_export(conn, tapepool, dryrun):
'''helper function to execute the export process in a separate thread'''
cur = conn.cursor()
cur.execute('ALTER SESSION ENABLE PARALLEL QUERY')
cur = conn.cursor()
cur.execute('ALTER SESSION ENABLE PARALLEL DML')
cur = conn.cursor()
cur.execute('BEGIN completeCTAExport(:tapepool, :dryrun); END;', \
tapepool=tapepool, dryrun=dryrun)
def run():
'''main code'''
success = False
dryrun = 0
tapepool = ''
# first parse the options
try:
options, _ = getopt.getopt(sys.argv[1:], 'ht:d', ['help', 'tapepool=', 'dryrun'])
except Exception, e:
print e
usage(1)
for f, v in options:
if f == '-h' or f == '--help':
usage(0)
elif f == '-t' or f == '--tapepool':
tapepool = v
elif f == '-d' or f == '--dryrun':
dryrun = 1
else:
print "unknown option : " + f
usage(1)
# deal with arguments
if tapepool == '':
print 'Missing argument(s)'
usage(1)
try:
# connect to the Nameserver and execute the complete_export procedure on a separate thread, to be able to babysit it
nsconn_async = castor_tools.connectToNS()
runner = Thread(target=async_complete_export, args=(nsconn_async, tapepool, dryrun))
runner.start()
# at the same time, connect again to the Nameserver for monitoring the process
sleep(1)
nsconn = castor_tools.connectToNS()
nscur = nsconn.cursor()
querylog = '''
SELECT timestamp, message FROM CTAMigrationLog
WHERE tapepool = :tapepool AND timestamp > :t ORDER BY timestamp ASC
'''
# poll the NS database for logs about the ongoing migration
t = 0
while True:
nscur.execute(querylog, tapepool=tapepool, t=t)
rows = nscur.fetchall()
for r in rows:
print datetime.fromtimestamp(int(r[0])).isoformat(), ' ', r[1]
if not rows:
# keep printing something when no news
print datetime.now().isoformat().split('.')[0], ' .'
elif 'Export from CASTOR fully completed' in rows[-1][1]:
# export is over, terminate
success = True
break
if rows:
t = rows[-1][0]
# exit also in case of premature termination
if not runner.isAlive():
break
sleep(60)
# that ought to be immediate now
runner.join()
# close DB connections
castor_tools.disconnectDB(nsconn_async)
castor_tools.disconnectDB(nsconn)
except Exception, e:
print e
import traceback
traceback.print_exc()
sys.exit(-1)
if __name__ == '__main__':
run()
#!/bin/sh
#!/bin/bash
#/******************************************************************************
# * exporttapepool
# * exporttapepool.sh
# *
# * This file is part of the Castor/CTA project.
# * See http://cern.ch/castor and http://cern.ch/eoscta
......@@ -66,8 +66,8 @@ python tapepool_castor_to_cta.py -t $1 -v $2 -i $3 $4 || exit 1
eos-import-dirs || exit 1
eos-import-files || exit 1
# terminate the import
python complete_tapepool_import.py $1 || exit 1
# terminate the export
python complete_tapepool_export.py -t $1 $4 || exit 1
# empty the CASTOR disk cache (not necessary)
#for h in `printdiskserver | grep cern.ch | awk '{print $1}'`; do
......
#!/bin/sh
#!/bin/bash
#/******************************************************************************
# * undoexporttapepool
# * undoexporttapepool.sh
# *
# * This file is part of the Castor/CTA project.
# * See http://cern.ch/castor and http://cern.ch/eoscta
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment