Skip to content
Snippets Groups Projects
Commit b5fcf53e authored by Vladimir Bahyl's avatar Vladimir Bahyl
Browse files

Adding scripts to generate TPCONFIG

parent 55f6cd30
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
operations/tape/tape-config-generate 0 → 100755
+ 193
− 0
View file @ b5fcf53e
#!/usr/bin/perl -w
#######################################################################
#
# This script will generate /etc/cta/TPCONFIG file with the data from
# TOMS (Tape Operations Management System) URL:
#
# https://apex.cern.ch/pls/htmldb_castorns/f?p=toms_prod:250:163672298908022::NO::P250_TAPESERVER:HOSTNAME
#
# Vladimir Bahyl - 05/2019
#
#######################################################################
use strict;
use XML::DOM;
use Sys::Hostname;
use LWP::UserAgent;
use LC::Check qw(file);
#use Data::Dumper;
my $today = localtime;
my %TPCONFIG = ();
my $hostname = '';
my $tpconfigfile = '/etc/castor/TPCONFIG';
my $tpconfig = "#######################################################################
#
# CTA Tape Server Configuration file
#
# This tape server is not configured.
#
#######################################################################
#
# Generated on $today by $0
";
my $changes = 0;
($hostname = hostname()) =~ s/\.cern\.ch$//io;
my $configUrl = 'https://apex.cern.ch/pls/htmldb_castorns/f?p=toms_prod:250:163672298908022::NO::P250_TAPESERVER:HOSTNAME';
die ("$0: missing configuration URL") unless ($configUrl);
$configUrl =~ s/HOSTNAME/$hostname/o;
#
# Fetch the data
#
print("$0: Fetching the data over HTTP from the Oracle APEX database ... please be patient ...\n");
%TPCONFIG = &GetData($configUrl);
#
# Prepare the TPCONFIG file
#
my $i = 0;
while (%TPCONFIG and defined($TPCONFIG{$i}{'tapeserver'}) and (lc($TPCONFIG{$i}{'tapeserver'}) eq lc($hostname))) {
$tpconfig = "#######################################################################
#
# CTA Tape Server Configuration file
#
# unit device system control
# name group device method
" if ($i == 0);
$tpconfig .= "$TPCONFIG{$i}{'tapedrive'} $TPCONFIG{$i}{'devicegroup'} $TPCONFIG{$i}{'unixdevice'} $TPCONFIG{$i}{'controlmethod'}
# Tape Drive Comment: $TPCONFIG{$i}{'tapedrivecomment'}
# Tape Service Comment: $TPCONFIG{$i}{'tapeservicecomment'}
# Modified by: $TPCONFIG{$i}{'modifuser'}
# Modify date: $TPCONFIG{$i}{'modifdate'}
";
$i++;
}
$tpconfig .= "#
#######################################################################
#
# Generated on $today by $0
" if (%TPCONFIG and (lc($TPCONFIG{0}{'tapeserver'}) eq lc($hostname)));
# Change the TPCONFIG location if comment mentions CTA
$tpconfigfile = '/etc/cta/TPCONFIG' if (%TPCONFIG and (defined $TPCONFIG{0}{'tapeservicecomment'}) and ($TPCONFIG{0}{'tapeservicecomment'} =~ /CTA/oi));
#
# Configure TPCONFIG and SSI files
#
$changes += &UpdateFile($tpconfigfile, $tpconfig);
LC::Check::link('/etc/TPCONFIG', $tpconfigfile,
backup => '.old',
nocheck => 1,
force => 1
);
##########################################################################
sub GetData {
##########################################################################
my ($url) = @_;
my %TPCONFIG = ();
my $xmlParser = new XML::DOM::Parser;
# Download the XML formated configuration data
# Use cookies because of APEX (otherwise, nothing will be downloaded; redirection will not work)
my $UserAgent = LWP::UserAgent->new;
$UserAgent->cookie_jar({ file => undef });
my $xml = $UserAgent->get($url);
if (defined ($xml->content)) {
if ($xml->content =~/TAPESERVER/oi) {
my $xmlDoc = $xmlParser->parse($xml->content);
# pcitpdp39 ~ > lynx -source "http://oraweb.cern.ch/pls/cdbsqldev/web.show_tpconfig?p_tapeserver=tpsrv027&p_output=xml"
# <?xml version = '1.0'?>
# <TPCONFIGSEARCHLIST> <TAPESERVER NAME="tpsrv027" TAPEDRIVE="994B53A6" DEVICEGROUP="994BR5" UNIXDEVICE="/dev/nst0" DENSITY="200G" COMPRESSION="Y" INITSTATUS="DOWN" CONTROLMETHOD="acs0,3,10,6" MODEL="9940" ROBOTHOST="sunstk62" /> </TPCONFIGSEARCHLIST>
for my $i (0 .. ($xmlDoc->getElementsByTagName("TAPESERVER")->getLength())-1) {
$TPCONFIG{$i}{'tapeserver'} = $xmlDoc->getElementsByTagName("TAPESERVER")->item($i)->getAttribute("NAME");
$TPCONFIG{$i}{'tapedrive'} = $xmlDoc->getElementsByTagName("TAPESERVER")->item($i)->getAttribute("TAPEDRIVE");
$TPCONFIG{$i}{'devicegroup'} = $xmlDoc->getElementsByTagName("TAPESERVER")->item($i)->getAttribute("DEVICEGROUP");
$TPCONFIG{$i}{'unixdevice'} = $xmlDoc->getElementsByTagName("TAPESERVER")->item($i)->getAttribute("UNIXDEVICE");
$TPCONFIG{$i}{'initstatus'} = lc($xmlDoc->getElementsByTagName("TAPESERVER")->item($i)->getAttribute("INITSTATUS"));
$TPCONFIG{$i}{'controlmethod'} = $xmlDoc->getElementsByTagName("TAPESERVER")->item($i)->getAttribute("CONTROLMETHOD");
$TPCONFIG{$i}{'modifdate'} = $xmlDoc->getElementsByTagName("TAPESERVER")->item($i)->getAttribute("MODIFDATE");
$TPCONFIG{$i}{'modifuser'} = $xmlDoc->getElementsByTagName("TAPESERVER")->item($i)->getAttribute("MODIFUSER");
$TPCONFIG{$i}{'tapedrivecomment'} = $xmlDoc->getElementsByTagName("TAPESERVER")->item($i)->getAttribute("TAPEDRIVECOMMENT");
$TPCONFIG{$i}{'tapeservicecomment'} = $xmlDoc->getElementsByTagName("TAPESERVER")->item($i)->getAttribute("TAPESERVICECOMMENT");
warn("$0: database entry nr. ".($i+1)." missing tape server hostname\n") unless ($TPCONFIG{$i}{'tapeserver'});
warn("$0: database entry nr. ".($i+1)." missing tape drive name\n") unless ($TPCONFIG{$i}{'tapedrive'});
warn("$0: database entry nr. ".($i+1)." missing device group name\n") unless ($TPCONFIG{$i}{'devicegroup'});
warn("$0: database entry nr. ".($i+1)." missing unix device\n") unless ($TPCONFIG{$i}{'unixdevice'});
warn("$0: database entry nr. ".($i+1)." missing init status\n") unless ($TPCONFIG{$i}{'initstatus'});
warn("$0: database entry nr. ".($i+1)." missing control method\n") unless ($TPCONFIG{$i}{'controlmethod'});
warn("$0: database entry nr. ".($i+1)." missing the modification date\n") unless ($TPCONFIG{$i}{'modifdate'});
warn("$0: database entry nr. ".($i+1)." missing user name\n") unless ($TPCONFIG{$i}{'modifuser'});
print("$0: database entry nr. ".($i+1)." no tape service comment\n") unless ($TPCONFIG{$i}{'tapeservicecomment'});
print("$0: database entry nr. ".($i+1)." no tape drive comment\n") unless ($TPCONFIG{$i}{'tapedrivecomment'});
}
$xmlDoc->dispose;
} else {
warn("$0: URL $url is not returning any usable data for $hostname. This tape server will not be configured. Please check whether there is a tape drive assigned to this tape server.\n");
}
} else {
warn("$0: URL $url doesn't seem to work. There could be a problem with the Web server or the Oracle APEX database server.\n");
}
return %TPCONFIG;
}
##########################################################################
sub UpdateFile {
##########################################################################
my ($filename, $newcontent) = @_;
my $changes = 0;
if ((-f $filename) and (-r $filename) and (-s $filename)) {
# Check the content of the file and correct it if there are some differences
$changes += LC::Check::file($filename,
source => $filename,
owner => 0,
group => 0,
mode => 0644,
backup => '.old',
code => sub {
my($oldcontent) = @_;
return() unless $oldcontent;
(my $oldfile = $oldcontent) =~ s/^#.*$//gim; # remove lines with comments
$oldfile =~ s/^\s*$//gim; # remove empty lines
(my $newfile = $newcontent) =~ s/^#.*$//gim; # remove lines with comments
$newfile =~ s/^\s*$//gim; # remove empty lines
$oldcontent = $newcontent unless ($oldfile eq $newfile);
return($oldcontent);
}
);
} else {
# The file is missing, create a new one
$changes += LC::File::file_contents($filename, $newcontent);
print "$0: created new $filename\n";
}
die ("$0: error modifying $filename\n") unless (defined($changes));
return $changes;
}
operations/tape/tape-devices-namer 0 → 100755
+ 227
− 0
View file @ b5fcf53e
#!/usr/bin/python
"""
This script creates links to tape and medium changer devices.
The association between tape and smc device is made based on
the serial numbers stored in the TOMS DB.
"""
import re
import os
import sys
import socket
import pprint
import urllib2
import optparse
import cookielib
import subprocess
pp = pprint.PrettyPrinter(width=200)
#------------------------------------------------------------
def mklink(dev, sn, drivename, type):
if not options.noaction and drivename is not None:
link = '/dev/' + type + '_' + drivename
subprocess.Popen(['/bin/ln', '-f', '-s', dev, link]).wait()
print 'Created link', link
else:
print 'Cannot create link to ' + dev
print 'Drivename for serial number ' + sn + ' not found'
#------------------------------------------------------------
def fix_mismatch(mm_tape_dev, toms_drives, hostname):
#this fucntions assumes that there is only one mismatch,
#i.e. only one tape device with S/N not found in TOMS
#and only one drives in TOSM with S/N not found in the server.
l = []
cj = cookielib.CookieJar()
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
for d in toms_drives:
if d['match'] == 0:
#assigning drivename to the mismatched tape device
mm_tape_dev['drivename'] = d['drivename']
#fixing the S/N in TOMS
tomsurl = 'http://castortapeweb.cern.ch/cgi-bin/serial-nr-update.cgi?tapedrive=' + d['drivename'] + '&tapeserver=' + hostname + '&serialnumber=' + mm_tape_dev['sn']
if options.debug: print 'Opening', tomsurl
try:
urlfh = opener.open(tomsurl)
except:
print 'Cannot open ' + tomsurl
#------------------------------------------------------------
#main
#------------------------------------------------------------
# options ---------------------------------------------------
usage = "usage: %prog [options]"
parser = optparse.OptionParser(usage)
parser.add_option("-d", "--debug", action="store_true", dest="debug", help="print debug messages")
parser.add_option("--noaction", action="store_true", dest="noaction", help="do nothing")
(options, args) = parser.parse_args()
tape_devices = []
smc_devices = []
# find tape and smc devices
if options.debug: print 'Searching tape devices'
try:
p = subprocess.Popen(['/usr/bin/lsscsi', '-g'], stdout=subprocess.PIPE)
p.wait()
except:
print 'Cannot run lsscsi. Exit.'
sys.exit(0)
for line in p.stdout:
fields = line.split()
scsi_address = fields[0][1:-1]
type = fields[1]
scsi_generic = fields.pop()
scsi_tape = fields.pop()
if type == 'tape':
tape_devices.append({'scsi_address' : scsi_address, 'scsi_generic' : scsi_generic, 'scsi_tape' : scsi_tape, 'sn' : None, 'drivename' : None})
if type == 'mediumx':
smc_devices.append({'scsi_address' : scsi_address, 'scsi_generic' : scsi_generic, 'scsi_tape' : scsi_tape, 'sn' : None, 'drivename' : None})
ntpdev=len(tape_devices)
if options.debug:
print 'tape_devices:'
pp.pprint(tape_devices)
print 'smc_devices:'
pp.pprint(smc_devices)
# associate tape and smc devices
if options.debug: print 'Coupling tape and smc devices (if any)'
pairs = []
for tapedev in tape_devices:
for smcdev in smc_devices:
if tapedev['scsi_address'][:-2] == smcdev['scsi_address'][:-2]:
pairs.append([tapedev, smcdev])
if options.debug:
print 'pairs:'
pp.pprint(pairs)
if len(tape_devices)>len(smc_devices) and len(smc_devices)>0:
print 'Number of control paths lower that number of drives'
sys.exit(0)
# find the serial number of the tape device
# sg_inq will not work if the /dev/sgX device is not reachable
# sg_inq will not work if the /dev/nstY device is being used
# run sg_inq against the nst dev so that if it is already being used we exit
if options.debug: print 'Reading serial numbers from tape devices'
for tapedev in tape_devices:
tapedn = '/dev/nst'+tapedev['scsi_tape'][-1]
p = subprocess.Popen(['/usr/bin/sg_inq', tapedn], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if p.wait() != 0:
print 'Cannot run sg_inq on ' + tapedn + '. Exit'
sys.exit(0)
else:
for line in p.stdout:
#reg = re.search('(?<=Unit serial number: )(\d+)', line)
if re.search('Unit serial number', line):
l = line.split(':')
tapedev['sn'] = l[1][1:-1]
if tapedev['sn'] is None:
print 'Could not extract the serial number from the output of sg_inq ' + tapedn
sys.exit(0)
if options.debug:
print 'tape_devices:'
pp.pprint(tape_devices)
# search the drive names in toms by serial number
toms_drives = []
if options.debug: print 'Looking into TOMS for drive names'
hostname = socket.gethostname().split('.')[0]
tomsurl = 'https://apex.cern.ch/pls/htmldb_castorns/f?p=toms_prod:250:::NO::P250_TAPESERVER:' + hostname
if options.debug: print 'Opening', tomsurl
cj = cookielib.CookieJar()
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
try:
urlfh = opener.open(tomsurl)
except:
print 'Cannot open ' + tomsurl
sys.exit(0)
for line in urlfh:
if options.debug: print line
if not re.search('TAPESERVER', line): continue
drivename, serialnumber = '', ''
l = line.split()
for item in l:
g = item.split('=')
if g[0] == 'CURRENTSERIALNR': serialnumber = g[1][1:-1]
if g[0] == 'TAPEDRIVE': drivename = g[1][1:-1]
if drivename == '':
print 'drive name for host ' + hostname + ' not found in TOMS'
sys.exit(0)
if serialnumber == '':
print 'Serial number for drive', drivename, 'not found in TOMS'
#here we don't exit, in case of a signle mismatch we update the s/n
toms_drives.append({'drivename' : drivename, 'sn': serialnumber, 'match' : 0})
for tapedev in tape_devices:
if tapedev['sn'] == serialnumber:
tapedev['drivename'] = drivename
for d in toms_drives:
if d['drivename'] == drivename: d['match'] = 1
if options.debug:
print 'tape_devices:'
pp.pprint(tape_devices)
print 'toms_drives:'
pp.pprint(toms_drives)
#Check how many S/N are missing.
#1. If there is only one assume that the drive has been replaced and update the S/N in TOMS.
#2. If there are more than one the script does nothing (new or changed drives/devices
# will not be configured (link not created).
devs_mm_sn = 0
mm_tape_dev = None
for t in tape_devices:
if t['drivename'] is None:
devs_mm_sn += 1
mm_tape_dev = t
if devs_mm_sn == 1:
print 'One S/N mismatch. Going to fix S/N in TOMS'
fix_mismatch(mm_tape_dev, toms_drives, hostname)
elif devs_mm_sn == 0:
if options.debug: print 'No S/N mismatches'
else:
if options.debug: print 'Too many S/N mismatches'
# created links
if pairs == []:
# this is a SUN tape server
for tapedev in tape_devices:
tapedn = '/dev/nst'+tapedev['scsi_tape'][-1]
mklink(tapedn, tapedev['sn'], tapedev['drivename'], 'tape')
else:
#this is a IBM tape server
for pair in pairs:
tapedev = pair[0]
tapedn = '/dev/nst'+tapedev['scsi_tape'][-1]
mklink(tapedn, tapedev['sn'], tapedev['drivename'], 'tape')
smcdev = pair[1]
mklink(smcdev['scsi_generic'], tapedev['sn'], tapedev['drivename'], 'smc')
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

imprint | privacy | Conditions of Use