diff --git a/operations/tape/tape-config-generate b/operations/tape/tape-config-generate new file mode 100755 index 0000000000000000000000000000000000000000..fcb5ff47998e6fe76b3e73c591a1ccab2e28fe0b --- /dev/null +++ b/operations/tape/tape-config-generate @@ -0,0 +1,193 @@ +#!/usr/bin/perl -w +####################################################################### +# +# This script will generate /etc/cta/TPCONFIG file with the data from +# TOMS (Tape Operations Management System) URL: +# +# https://apex.cern.ch/pls/htmldb_castorns/f?p=toms_prod:250:163672298908022::NO::P250_TAPESERVER:HOSTNAME +# +# Vladimir Bahyl - 05/2019 +# +####################################################################### + +use strict; +use XML::DOM; +use Sys::Hostname; +use LWP::UserAgent; +use LC::Check qw(file); + +#use Data::Dumper; + +my $today = localtime; + +my %TPCONFIG = (); +my $hostname = ''; + +my $tpconfigfile = '/etc/castor/TPCONFIG'; +my $tpconfig = "####################################################################### +# +# CTA Tape Server Configuration file +# +# This tape server is not configured. +# +####################################################################### +# +# Generated on $today by $0 +"; + +my $changes = 0; + +($hostname = hostname()) =~ s/\.cern\.ch$//io; + +my $configUrl = 'https://apex.cern.ch/pls/htmldb_castorns/f?p=toms_prod:250:163672298908022::NO::P250_TAPESERVER:HOSTNAME'; +die ("$0: missing configuration URL") unless ($configUrl); +$configUrl =~ s/HOSTNAME/$hostname/o; + +# +# Fetch the data +# +print("$0: Fetching the data over HTTP from the Oracle APEX database ... please be patient ...\n"); +%TPCONFIG = &GetData($configUrl); + +# +# Prepare the TPCONFIG file +# +my $i = 0; +while (%TPCONFIG and defined($TPCONFIG{$i}{'tapeserver'}) and (lc($TPCONFIG{$i}{'tapeserver'}) eq lc($hostname))) { + $tpconfig = "####################################################################### +# +# CTA Tape Server Configuration file +# +# unit device system control +# name group device method + +" if ($i == 0); + + $tpconfig .= "$TPCONFIG{$i}{'tapedrive'} $TPCONFIG{$i}{'devicegroup'} $TPCONFIG{$i}{'unixdevice'} $TPCONFIG{$i}{'controlmethod'} + +# Tape Drive Comment: $TPCONFIG{$i}{'tapedrivecomment'} +# Tape Service Comment: $TPCONFIG{$i}{'tapeservicecomment'} +# Modified by: $TPCONFIG{$i}{'modifuser'} +# Modify date: $TPCONFIG{$i}{'modifdate'} + +"; + $i++; +} + +$tpconfig .= "# +####################################################################### +# +# Generated on $today by $0 +" if (%TPCONFIG and (lc($TPCONFIG{0}{'tapeserver'}) eq lc($hostname))); + +# Change the TPCONFIG location if comment mentions CTA +$tpconfigfile = '/etc/cta/TPCONFIG' if (%TPCONFIG and (defined $TPCONFIG{0}{'tapeservicecomment'}) and ($TPCONFIG{0}{'tapeservicecomment'} =~ /CTA/oi)); + +# +# Configure TPCONFIG and SSI files +# +$changes += &UpdateFile($tpconfigfile, $tpconfig); + +LC::Check::link('/etc/TPCONFIG', $tpconfigfile, + backup => '.old', + nocheck => 1, + force => 1 +); + +########################################################################## +sub GetData { +########################################################################## + my ($url) = @_; + + my %TPCONFIG = (); + + my $xmlParser = new XML::DOM::Parser; + + # Download the XML formated configuration data + # Use cookies because of APEX (otherwise, nothing will be downloaded; redirection will not work) + my $UserAgent = LWP::UserAgent->new; + $UserAgent->cookie_jar({ file => undef }); + my $xml = $UserAgent->get($url); + + if (defined ($xml->content)) { + if ($xml->content =~/TAPESERVER/oi) { + my $xmlDoc = $xmlParser->parse($xml->content); + + # pcitpdp39 ~ > lynx -source "http://oraweb.cern.ch/pls/cdbsqldev/web.show_tpconfig?p_tapeserver=tpsrv027&p_output=xml" + # <?xml version = '1.0'?> + # <TPCONFIGSEARCHLIST> <TAPESERVER NAME="tpsrv027" TAPEDRIVE="994B53A6" DEVICEGROUP="994BR5" UNIXDEVICE="/dev/nst0" DENSITY="200G" COMPRESSION="Y" INITSTATUS="DOWN" CONTROLMETHOD="acs0,3,10,6" MODEL="9940" ROBOTHOST="sunstk62" /> </TPCONFIGSEARCHLIST> + + for my $i (0 .. ($xmlDoc->getElementsByTagName("TAPESERVER")->getLength())-1) { + $TPCONFIG{$i}{'tapeserver'} = $xmlDoc->getElementsByTagName("TAPESERVER")->item($i)->getAttribute("NAME"); + $TPCONFIG{$i}{'tapedrive'} = $xmlDoc->getElementsByTagName("TAPESERVER")->item($i)->getAttribute("TAPEDRIVE"); + $TPCONFIG{$i}{'devicegroup'} = $xmlDoc->getElementsByTagName("TAPESERVER")->item($i)->getAttribute("DEVICEGROUP"); + $TPCONFIG{$i}{'unixdevice'} = $xmlDoc->getElementsByTagName("TAPESERVER")->item($i)->getAttribute("UNIXDEVICE"); + $TPCONFIG{$i}{'initstatus'} = lc($xmlDoc->getElementsByTagName("TAPESERVER")->item($i)->getAttribute("INITSTATUS")); + $TPCONFIG{$i}{'controlmethod'} = $xmlDoc->getElementsByTagName("TAPESERVER")->item($i)->getAttribute("CONTROLMETHOD"); + $TPCONFIG{$i}{'modifdate'} = $xmlDoc->getElementsByTagName("TAPESERVER")->item($i)->getAttribute("MODIFDATE"); + $TPCONFIG{$i}{'modifuser'} = $xmlDoc->getElementsByTagName("TAPESERVER")->item($i)->getAttribute("MODIFUSER"); + $TPCONFIG{$i}{'tapedrivecomment'} = $xmlDoc->getElementsByTagName("TAPESERVER")->item($i)->getAttribute("TAPEDRIVECOMMENT"); + $TPCONFIG{$i}{'tapeservicecomment'} = $xmlDoc->getElementsByTagName("TAPESERVER")->item($i)->getAttribute("TAPESERVICECOMMENT"); + + warn("$0: database entry nr. ".($i+1)." missing tape server hostname\n") unless ($TPCONFIG{$i}{'tapeserver'}); + warn("$0: database entry nr. ".($i+1)." missing tape drive name\n") unless ($TPCONFIG{$i}{'tapedrive'}); + warn("$0: database entry nr. ".($i+1)." missing device group name\n") unless ($TPCONFIG{$i}{'devicegroup'}); + warn("$0: database entry nr. ".($i+1)." missing unix device\n") unless ($TPCONFIG{$i}{'unixdevice'}); + warn("$0: database entry nr. ".($i+1)." missing init status\n") unless ($TPCONFIG{$i}{'initstatus'}); + warn("$0: database entry nr. ".($i+1)." missing control method\n") unless ($TPCONFIG{$i}{'controlmethod'}); + warn("$0: database entry nr. ".($i+1)." missing the modification date\n") unless ($TPCONFIG{$i}{'modifdate'}); + warn("$0: database entry nr. ".($i+1)." missing user name\n") unless ($TPCONFIG{$i}{'modifuser'}); + print("$0: database entry nr. ".($i+1)." no tape service comment\n") unless ($TPCONFIG{$i}{'tapeservicecomment'}); + print("$0: database entry nr. ".($i+1)." no tape drive comment\n") unless ($TPCONFIG{$i}{'tapedrivecomment'}); + } + + $xmlDoc->dispose; + } else { + warn("$0: URL $url is not returning any usable data for $hostname. This tape server will not be configured. Please check whether there is a tape drive assigned to this tape server.\n"); + } + } else { + warn("$0: URL $url doesn't seem to work. There could be a problem with the Web server or the Oracle APEX database server.\n"); + } + + return %TPCONFIG; +} + +########################################################################## +sub UpdateFile { +########################################################################## + my ($filename, $newcontent) = @_; + + my $changes = 0; + + if ((-f $filename) and (-r $filename) and (-s $filename)) { + # Check the content of the file and correct it if there are some differences + $changes += LC::Check::file($filename, + source => $filename, + owner => 0, + group => 0, + mode => 0644, + backup => '.old', + code => sub { + my($oldcontent) = @_; + return() unless $oldcontent; + + (my $oldfile = $oldcontent) =~ s/^#.*$//gim; # remove lines with comments + $oldfile =~ s/^\s*$//gim; # remove empty lines + + (my $newfile = $newcontent) =~ s/^#.*$//gim; # remove lines with comments + $newfile =~ s/^\s*$//gim; # remove empty lines + + $oldcontent = $newcontent unless ($oldfile eq $newfile); + + return($oldcontent); + } + ); + } else { + # The file is missing, create a new one + $changes += LC::File::file_contents($filename, $newcontent); + print "$0: created new $filename\n"; + } + die ("$0: error modifying $filename\n") unless (defined($changes)); + + return $changes; +} diff --git a/operations/tape/tape-devices-namer b/operations/tape/tape-devices-namer new file mode 100755 index 0000000000000000000000000000000000000000..708bea830bd59e1d3f8d4002363092e72fb6d892 --- /dev/null +++ b/operations/tape/tape-devices-namer @@ -0,0 +1,227 @@ +#!/usr/bin/python + +""" +This script creates links to tape and medium changer devices. +The association between tape and smc device is made based on +the serial numbers stored in the TOMS DB. +""" + +import re +import os +import sys +import socket +import pprint +import urllib2 +import optparse +import cookielib +import subprocess +pp = pprint.PrettyPrinter(width=200) + +#------------------------------------------------------------ +def mklink(dev, sn, drivename, type): + if not options.noaction and drivename is not None: + link = '/dev/' + type + '_' + drivename + subprocess.Popen(['/bin/ln', '-f', '-s', dev, link]).wait() + print 'Created link', link + else: + print 'Cannot create link to ' + dev + print 'Drivename for serial number ' + sn + ' not found' + +#------------------------------------------------------------ +def fix_mismatch(mm_tape_dev, toms_drives, hostname): + #this fucntions assumes that there is only one mismatch, + #i.e. only one tape device with S/N not found in TOMS + #and only one drives in TOSM with S/N not found in the server. + l = [] + cj = cookielib.CookieJar() + opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj)) + for d in toms_drives: + if d['match'] == 0: + #assigning drivename to the mismatched tape device + mm_tape_dev['drivename'] = d['drivename'] + #fixing the S/N in TOMS + tomsurl = 'http://castortapeweb.cern.ch/cgi-bin/serial-nr-update.cgi?tapedrive=' + d['drivename'] + '&tapeserver=' + hostname + '&serialnumber=' + mm_tape_dev['sn'] + if options.debug: print 'Opening', tomsurl + try: + urlfh = opener.open(tomsurl) + except: + print 'Cannot open ' + tomsurl + + +#------------------------------------------------------------ +#main +#------------------------------------------------------------ + +# options --------------------------------------------------- +usage = "usage: %prog [options]" +parser = optparse.OptionParser(usage) +parser.add_option("-d", "--debug", action="store_true", dest="debug", help="print debug messages") +parser.add_option("--noaction", action="store_true", dest="noaction", help="do nothing") +(options, args) = parser.parse_args() + +tape_devices = [] +smc_devices = [] + +# find tape and smc devices +if options.debug: print 'Searching tape devices' +try: + p = subprocess.Popen(['/usr/bin/lsscsi', '-g'], stdout=subprocess.PIPE) + p.wait() +except: + print 'Cannot run lsscsi. Exit.' + sys.exit(0) + +for line in p.stdout: + fields = line.split() + scsi_address = fields[0][1:-1] + type = fields[1] + scsi_generic = fields.pop() + scsi_tape = fields.pop() + if type == 'tape': + tape_devices.append({'scsi_address' : scsi_address, 'scsi_generic' : scsi_generic, 'scsi_tape' : scsi_tape, 'sn' : None, 'drivename' : None}) + if type == 'mediumx': + smc_devices.append({'scsi_address' : scsi_address, 'scsi_generic' : scsi_generic, 'scsi_tape' : scsi_tape, 'sn' : None, 'drivename' : None}) + +ntpdev=len(tape_devices) + +if options.debug: + print 'tape_devices:' + pp.pprint(tape_devices) + print 'smc_devices:' + pp.pprint(smc_devices) + +# associate tape and smc devices +if options.debug: print 'Coupling tape and smc devices (if any)' +pairs = [] +for tapedev in tape_devices: + for smcdev in smc_devices: + if tapedev['scsi_address'][:-2] == smcdev['scsi_address'][:-2]: + pairs.append([tapedev, smcdev]) +if options.debug: + print 'pairs:' + pp.pprint(pairs) + +if len(tape_devices)>len(smc_devices) and len(smc_devices)>0: + print 'Number of control paths lower that number of drives' + sys.exit(0) + + +# find the serial number of the tape device +# sg_inq will not work if the /dev/sgX device is not reachable +# sg_inq will not work if the /dev/nstY device is being used +# run sg_inq against the nst dev so that if it is already being used we exit +if options.debug: print 'Reading serial numbers from tape devices' + +for tapedev in tape_devices: + tapedn = '/dev/nst'+tapedev['scsi_tape'][-1] + p = subprocess.Popen(['/usr/bin/sg_inq', tapedn], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if p.wait() != 0: + print 'Cannot run sg_inq on ' + tapedn + '. Exit' + sys.exit(0) + else: + for line in p.stdout: + #reg = re.search('(?<=Unit serial number: )(\d+)', line) + if re.search('Unit serial number', line): + l = line.split(':') + tapedev['sn'] = l[1][1:-1] + + if tapedev['sn'] is None: + print 'Could not extract the serial number from the output of sg_inq ' + tapedn + sys.exit(0) + +if options.debug: + print 'tape_devices:' + pp.pprint(tape_devices) + + +# search the drive names in toms by serial number +toms_drives = [] +if options.debug: print 'Looking into TOMS for drive names' +hostname = socket.gethostname().split('.')[0] +tomsurl = 'https://apex.cern.ch/pls/htmldb_castorns/f?p=toms_prod:250:::NO::P250_TAPESERVER:' + hostname +if options.debug: print 'Opening', tomsurl + +cj = cookielib.CookieJar() +opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj)) + +try: + urlfh = opener.open(tomsurl) +except: + print 'Cannot open ' + tomsurl + sys.exit(0) + +for line in urlfh: + + if options.debug: print line + + if not re.search('TAPESERVER', line): continue + + drivename, serialnumber = '', '' + + l = line.split() + for item in l: + g = item.split('=') + if g[0] == 'CURRENTSERIALNR': serialnumber = g[1][1:-1] + if g[0] == 'TAPEDRIVE': drivename = g[1][1:-1] + + if drivename == '': + print 'drive name for host ' + hostname + ' not found in TOMS' + sys.exit(0) + + if serialnumber == '': + print 'Serial number for drive', drivename, 'not found in TOMS' + #here we don't exit, in case of a signle mismatch we update the s/n + + toms_drives.append({'drivename' : drivename, 'sn': serialnumber, 'match' : 0}) + + for tapedev in tape_devices: + if tapedev['sn'] == serialnumber: + tapedev['drivename'] = drivename + for d in toms_drives: + if d['drivename'] == drivename: d['match'] = 1 + +if options.debug: + print 'tape_devices:' + pp.pprint(tape_devices) + print 'toms_drives:' + pp.pprint(toms_drives) + + +#Check how many S/N are missing. +#1. If there is only one assume that the drive has been replaced and update the S/N in TOMS. +#2. If there are more than one the script does nothing (new or changed drives/devices +# will not be configured (link not created). + +devs_mm_sn = 0 +mm_tape_dev = None +for t in tape_devices: + if t['drivename'] is None: + devs_mm_sn += 1 + mm_tape_dev = t + +if devs_mm_sn == 1: + print 'One S/N mismatch. Going to fix S/N in TOMS' + fix_mismatch(mm_tape_dev, toms_drives, hostname) +elif devs_mm_sn == 0: + if options.debug: print 'No S/N mismatches' +else: + if options.debug: print 'Too many S/N mismatches' + + +# created links +if pairs == []: + # this is a SUN tape server + for tapedev in tape_devices: + tapedn = '/dev/nst'+tapedev['scsi_tape'][-1] + mklink(tapedn, tapedev['sn'], tapedev['drivename'], 'tape') +else: + #this is a IBM tape server + for pair in pairs: + tapedev = pair[0] + tapedn = '/dev/nst'+tapedev['scsi_tape'][-1] + mklink(tapedn, tapedev['sn'], tapedev['drivename'], 'tape') + smcdev = pair[1] + mklink(smcdev['scsi_generic'], tapedev['sn'], tapedev['drivename'], 'smc') + + +