[migration] #576: added wrapper to populate the TLDs helper table

cta.spec.in

@@ -313,8 +313,10 @@ directory metadata into the EOS namespace.
 %attr(0755,root,root) %{_bindir}/eos-test-inject.sh
 %attr(0755,root,root) %{_bindir}/eos-insert-missing-dirs
 %attr(0755,root,root) %{_bindir}/json-pretty-print.sh
+%attr(0755,root,root) %{_bindir}/startvoexport.sh
 %attr(0755,root,root) %{_bindir}/exporttapepool.sh
 %attr(0755,root,root) %{_bindir}/undoexporttapepool.sh
+%attr(0755,root,root) %{_bindir}/dirs_castor_to_cta.py
 %attr(0755,root,root) %{_bindir}/tapepool_castor_to_cta.py
 %attr(0755,root,root) %{_bindir}/complete_tapepool_export.py
 %attr(0644,root,root) %config(noreplace) %{_sysconfdir}/cta/castor-migration.conf.example

migration/castor/CMakeLists.txt

@@ -14,8 +14,11 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-install(FILES ${CMAKE_SOURCE_DIR}/migration/castor/exporttapepool.sh
-              ${CMAKE_SOURCE_DIR}/migration/castor/undoexporttapepool.sh
-              ${CMAKE_SOURCE_DIR}/migration/castor/tapepool_castor_to_cta.py
-              ${CMAKE_SOURCE_DIR}/migration/castor/complete_tapepool_export.py
-        DESTINATION usr/bin)
+install(FILES
+    ${CMAKE_SOURCE_DIR}/migration/castor/startvoexport.sh
+    ${CMAKE_SOURCE_DIR}/migration/castor/exporttapepool.sh
+    ${CMAKE_SOURCE_DIR}/migration/castor/undoexporttapepool.sh
+    ${CMAKE_SOURCE_DIR}/migration/castor/dirs_castor_to_cta.py
+    ${CMAKE_SOURCE_DIR}/migration/castor/tapepool_castor_to_cta.py
+    ${CMAKE_SOURCE_DIR}/migration/castor/complete_tapepool_export.py
+    DESTINATION usr/bin)

migration/castor/dirs_castor_to_cta.py

+#!/usr/bin/python
+#/******************************************************************************
+# *                      dirs_castor_to_cta.py
+# *
+# * This file is part of the Castor/CTA project.
+# * See http://cern.ch/castor and http://cern.ch/eoscta
+# * Copyright (C) 2019  CERN
+# *
+# * This program is free software; you can redistribute it and/or
+# * modify it under the terms of the GNU General Public License
+# * as published by the Free Software Foundation; either version 2
+# * of the License, or (at your option) any later version.
+# * This program is distributed in the hope that it will be useful,
+# * but WITHOUT ANY WARRANTY; without even the implied warranty of
+# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# * GNU General Public License for more details.
+# * You should have received a copy of the GNU General Public License
+# * along with this program; if not, write to the Free Software
+# * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+# *
+# * @author Castor Dev team, castor-dev@cern.ch
+# *****************************************************************************/
+
+'''command line tool to inject the top-level dirs in the CASTOR Nameserver for a subsequent CASTOR to CTA export'''
+
+import sys
+import getopt
+
+import castor_tools
+
+# usage function
+def usage(exitcode):
+    '''prints usage'''
+    print 'Usage : ' + sys.argv[0] + ' [-h|--help] -v|--vo <VO> [-t|--truncate] list of top-level dirs'
+    sys.exit(exitcode)
+
+
+def run():
+    '''main code'''
+    truncate = 0
+    vo = ''
+    # first parse the options
+    try:
+        options, args = getopt.getopt(sys.argv[1:], 'hv:t', ['help', 'vo=', 'truncate'])
+    except Exception, e:
+        print e
+        usage(1)
+    for f, v in options:
+        if f == '-h' or f == '--help':
+            usage(0)
+        elif f == '-v' or f == '--vo':
+            vo = v
+        elif f == '-t' or f == '--truncate':
+            truncate = 1
+        else:
+            print "unknown option : " + f
+            usage(1)
+
+    # deal with arguments
+    if vo == '' or len(args) == 0:
+        print 'Missing argument(s)'
+        usage(1)
+
+    try:
+        # connect to the Nameserver
+        nsconn = castor_tools.connectToNS()
+
+        if truncate:
+            cur = nsconn.cursor()
+            cur.execute('DELETE FROM CTATopLevelDirsHelper')
+        # insert the TLDs one by one
+        for d in args:
+            cur = nsconn.cursor()
+            cur.execute('INSERT INTO CTATopLevelDirsHelper VALUES (:dir)', dir=d)
+        nsconn.commit()
+
+        # insert also a log message
+        cur = nsconn.cursor()
+        cur.execute('BEGIN ctaLog(:tapepool, :msg); END;', \
+                    tapepool='{}::*'.format(vo), \
+                    msg='Added top-level directories to prepare a CTA import')
+
+        # close DB connection
+        castor_tools.disconnectDB(nsconn)
+
+    except Exception, e:
+        print e
+        import traceback
+        traceback.print_exc()
+        sys.exit(-1)
+
+
+if __name__ == '__main__':
+    run()

migration/castor/exporttapepool.sh

@@ -27,11 +27,11 @@
 # check arguments
 if [[ $# != 3 && $# != 4 ]]; then
   echo usage: $0 tapepool vo eosctaInstance [-d]
-  echo        -d enables dry-run mode
+  echo        -d  enables dry-run mode
   exit 1
 fi
 
-# if NOT dry-run, execute all pre-checks
+# if NOT dry-run, execute all pre-checks and drop the migration routes
 if [[ "$4" != "-d" ]]; then
 
   # check that the tapepool exists for this stager
@@ -45,40 +45,20 @@ if [[ "$4" != "-d" ]]; then
   busytapes=`vmgrlisttape -P $1 | grep -c BUSY`
   [[ $busytapes -gt 0 ]] && echo 'Found' $busytapes 'tape(s) in BUSY state, aborting' && exit 1
 
-  # backup relevant metadata
-  mkdir -p ~/ctaexport
-  cd ~/ctaexport
-  [[ ! -x stager_listprivileges_output ]] && \
-    stager_listprivileges > stager_listprivileges_output && \
-    # pause the stager altogether (this destroys the B&W lists!)
-    stager_removeprivilege -U:
-
   # on the stager, make the tapepool unusable (the tapepool metadata can stay)
-  [[ ! -x migrationroute_$1 ]] && printmigrationroute | grep -w $1 > migrationroutes_$1
+  [[ ! -x migrationroute_$1 ]] && printmigrationroute | grep -w $1 > ~/ctaexport/migrationroutes_$1
   printmigrationroute | grep -w $1 | awk '{print $1}' | xargs -i deletemigrationroute {}
-
 fi
 
-# exit for any error from any command
+# from now on, exit for any error from any command
 set -e
 
 # execute the DB extraction from the CTA DB
 tapepool_castor_to_cta.py -t $1 -v $2 -i $3 $4
 
 # execute the EOS metadata import
-eos-insert-missing-dirs
-eos-import-dirs
 eos-import-files
 
 # terminate the export
 complete_tapepool_export.py -t $1 $4
 
-# empty the CASTOR disk cache (not necessary)
-#for h in `printdiskserver | grep cern.ch | awk '{print $1}'`; do
-#	for f in `seq -w 1 24`; do
-#		deletediskcopy $h:/srv/castor/$f/
-#	done
-#done
-
-# to resume the stager once everything is completed:
-#stager_addprivilege -U:

migration/castor/startvoexport.sh

+#!/bin/bash
+#/******************************************************************************
+# *                      startvoexport.sh
+# *
+# * This file is part of the Castor/CTA project.
+# * See http://cern.ch/castor and http://cern.ch/eoscta
+# * Copyright (C) 2019  CERN
+# *
+# * This program is free software; you can redistribute it and/or
+# * modify it under the terms of the GNU General Public License
+# * as published by the Free Software Foundation; either version 2
+# * of the License, or (at your option) any later version.
+# * This program is distributed in the hope that it will be useful,
+# * but WITHOUT ANY WARRANTY; without even the implied warranty of
+# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# * GNU General Public License for more details.
+# * You should have received a copy of the GNU General Public License
+# * along with this program; if not, write to the Free Software
+# * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+# *
+# * command line tool to prepare the export of a tapepool metadata
+# * to CTA, the CASTOR successor
+#
+# * @author Castor Dev team, castor-dev@cern.ch
+# *****************************************************************************/
+
+# exit for any error from any command
+set -e
+
+print_usage() {
+  echo usage: $0 vo [-d] [-t] dir1 dir2 ...
+  echo '      -d  enables dry-run mode'
+  echo '      -t  truncates existing top-level dirs'
+  exit 1
+}
+
+# parse arguments
+if [[ $# -eq 0 ]]; then
+  print_usage
+fi
+vo=$1
+shift
+while [[ $# -gt 0 ]]; do
+  arg=$1
+  case $arg in
+    -d) dryrun=1; shift;;
+    -t) trunc=-t; shift;;
+    *)  break;;
+  esac
+done
+if [[ "$vo" == "" || "$@" == "" ]]; then
+  print_usage
+fi
+
+# if NOT dry-run
+if [[ "$dryrun" == "" ]]; then
+
+  # backup relevant metadata
+  mkdir -p ~/ctaexport
+  cd ~/ctaexport
+  [[ ! -x stager_listprivileges_output ]] && \
+    stager_listprivileges > stager_listprivileges_output && \
+    # pause the stager altogether (this destroys the B&W lists!)
+    stager_removeprivilege -U:
+fi
+
+# execute the DB extraction from the CTA DB
+./dirs_castor_to_cta.py -v $vo $trunc $@
+
+# execute the EOS metadata import
+eos-import-dirs
+
+# empty the CASTOR disk cache (not necessary)
+#for h in `printdiskserver | grep cern.ch | awk '{print $1}'`; do
+# for f in `seq -w 1 24`; do
+#   deletediskcopy $h:/srv/castor/$f/
+# done
+#done
+
+# to resume the stager once everything is completed:
+#stager_addprivilege -U:

migration/castor/undoexporttapepool.sh

@@ -38,3 +38,4 @@ cat ~/ctaexport/migrationroutes_$1 | grep -v FILECLASS | grep -v '---' | awk '{p
 
 # on the VMGR, mark all tapes back as available (but full == read-only) for the tape pool
 vmgrlisttape -P $1 | awk '{print $1}' | xargs -i vmgrmodifytape -V {} --st TAPE_FULL
+