Commit 591c5c48 authored by Vladimir Bahyl's avatar Vladimir Bahyl
Browse files

First attempt to create the utils RPM

parent 4f71dc5f
SPECFILE = cta-cern-tape-utils.spec
FILES = tape-config-generate
rpmtopdir := $(shell rpm --eval %_topdir)
rpmbuild := $(shell [ -x /usr/bin/rpmbuild ] && echo rpmbuild || echo rpm)
PKGNAME = $(shell grep -s '^Name:' $(SPECFILE) | sed -e 's/Name: *//')
PKGVERS = $(shell grep -s '^Version:' $(SPECFILE) | sed -e 's/Version: *//')
rpm: $(SPECFILE)
tar cvfz $(rpmtopdir)/SOURCES/$(PKGNAME)-$(PKGVERS).tgz $(FILES)
cp -f $(SPECFILE) $(rpmtopdir)/SPECS/$(SPECFILE)
$(rpmbuild) -ba $(SPECFILE)
Summary: CTA CERN specific tape utilities
Name: cta-cern-tape-utils
Version: 1.0
Release: 1
License: GPL
Buildroot: /tmp/%{name}-%{version}
BuildArch: noarch
Group: Application/CTA
Requires: python36 cta-cli
Source: %{name}-%{version}.tgz
%description
CTA CERN specific tape utilities:
- generate /etc/cta/TPCONFIG file from TOMS framework
- tape mount/unmount script
- tape label wrapper
- tape media check
- tape drive test
Author: David Fernandez Alvarez, Vladimir Bahyl - 8/2019
%prep
%setup -c
%install
[ -d $RPM_BUILD_ROOT ] && rm -rf $RPM_BUILD_ROOT
mkdir $RPM_BUILD_ROOT
cd $RPM_BUILD_ROOT
mkdir -p usr/local/bin etc/cron.d
name=`echo %{name} |sed 's/CERN-CC-//'`
install -m 755 $RPM_BUILD_DIR/%{name}-%{version}/tape-config-generate usr/local/bin/tape-config-generate
%post
%clean
rm -rf $RPM_BUILD_ROOT
%files
%defattr(-,root,root)
/usr/local/bin/tape-config-generate
#!/usr/bin/perl -w
#######################################################################
#
# This script will generate /etc/cta/TPCONFIG file with the data from
# TOMS (Tape Operations Management System) URL:
#
# https://apex.cern.ch/pls/htmldb_castorns/f?p=toms_prod:250:163672298908022::NO::P250_TAPESERVER:HOSTNAME
#
# Vladimir Bahyl - 05/2019
#
#######################################################################
use strict;
use XML::DOM;
use Sys::Hostname;
use LWP::UserAgent;
use LC::Check qw(file);
#use Data::Dumper;
my $today = localtime;
my %TPCONFIG = ();
my $hostname = '';
my $tpconfigfile = '/etc/castor/TPCONFIG';
my $tpconfig = "#######################################################################
#
# CTA Tape Server Configuration file
#
# This tape server is not configured.
#
#######################################################################
#
# Generated on $today by $0
";
my $changes = 0;
($hostname = hostname()) =~ s/\.cern\.ch$//io;
my $configUrl = 'https://apex.cern.ch/pls/htmldb_castorns/f?p=toms_prod:250:163672298908022::NO::P250_TAPESERVER:HOSTNAME';
die ("$0: missing configuration URL") unless ($configUrl);
$configUrl =~ s/HOSTNAME/$hostname/o;
#
# Fetch the data
#
print("$0: Fetching the data over HTTP from the Oracle APEX database ... please be patient ...\n");
%TPCONFIG = &GetData($configUrl);
#
# Prepare the TPCONFIG file
#
my $i = 0;
while (%TPCONFIG and defined($TPCONFIG{$i}{'tapeserver'}) and (lc($TPCONFIG{$i}{'tapeserver'}) eq lc($hostname))) {
$tpconfig = "#######################################################################
#
# CTA Tape Server Configuration file
#
# unit device system control
# name group device method
" if ($i == 0);
$tpconfig .= "$TPCONFIG{$i}{'tapedrive'} $TPCONFIG{$i}{'devicegroup'} $TPCONFIG{$i}{'unixdevice'} $TPCONFIG{$i}{'controlmethod'}
# Tape Drive Comment: $TPCONFIG{$i}{'tapedrivecomment'}
# Tape Service Comment: $TPCONFIG{$i}{'tapeservicecomment'}
# Modified by: $TPCONFIG{$i}{'modifuser'}
# Modify date: $TPCONFIG{$i}{'modifdate'}
";
$i++;
}
$tpconfig .= "#
#######################################################################
#
# Generated on $today by $0
" if (%TPCONFIG and (lc($TPCONFIG{0}{'tapeserver'}) eq lc($hostname)));
# Change the TPCONFIG location if comment mentions CTA
$tpconfigfile = '/etc/cta/TPCONFIG' if (%TPCONFIG and (defined $TPCONFIG{0}{'tapeservicecomment'}) and ($TPCONFIG{0}{'tapeservicecomment'} =~ /CTA/oi));
#
# Configure TPCONFIG and SSI files
#
$changes += &UpdateFile($tpconfigfile, $tpconfig);
LC::Check::link('/etc/TPCONFIG', $tpconfigfile,
backup => '.old',
nocheck => 1,
force => 1
);
##########################################################################
sub GetData {
##########################################################################
my ($url) = @_;
my %TPCONFIG = ();
my $xmlParser = new XML::DOM::Parser;
# Download the XML formated configuration data
# Use cookies because of APEX (otherwise, nothing will be downloaded; redirection will not work)
my $UserAgent = LWP::UserAgent->new;
$UserAgent->cookie_jar({ file => undef });
my $xml = $UserAgent->get($url);
if (defined ($xml->content)) {
if ($xml->content =~/TAPESERVER/oi) {
my $xmlDoc = $xmlParser->parse($xml->content);
# pcitpdp39 ~ > lynx -source "http://oraweb.cern.ch/pls/cdbsqldev/web.show_tpconfig?p_tapeserver=tpsrv027&p_output=xml"
# <?xml version = '1.0'?>
# <TPCONFIGSEARCHLIST> <TAPESERVER NAME="tpsrv027" TAPEDRIVE="994B53A6" DEVICEGROUP="994BR5" UNIXDEVICE="/dev/nst0" DENSITY="200G" COMPRESSION="Y" INITSTATUS="DOWN" CONTROLMETHOD="acs0,3,10,6" MODEL="9940" ROBOTHOST="sunstk62" /> </TPCONFIGSEARCHLIST>
for my $i (0 .. ($xmlDoc->getElementsByTagName("TAPESERVER")->getLength())-1) {
$TPCONFIG{$i}{'tapeserver'} = $xmlDoc->getElementsByTagName("TAPESERVER")->item($i)->getAttribute("NAME");
$TPCONFIG{$i}{'tapedrive'} = $xmlDoc->getElementsByTagName("TAPESERVER")->item($i)->getAttribute("TAPEDRIVE");
$TPCONFIG{$i}{'devicegroup'} = $xmlDoc->getElementsByTagName("TAPESERVER")->item($i)->getAttribute("DEVICEGROUP");
$TPCONFIG{$i}{'unixdevice'} = $xmlDoc->getElementsByTagName("TAPESERVER")->item($i)->getAttribute("UNIXDEVICE");
$TPCONFIG{$i}{'initstatus'} = lc($xmlDoc->getElementsByTagName("TAPESERVER")->item($i)->getAttribute("INITSTATUS"));
$TPCONFIG{$i}{'controlmethod'} = $xmlDoc->getElementsByTagName("TAPESERVER")->item($i)->getAttribute("CONTROLMETHOD");
$TPCONFIG{$i}{'modifdate'} = $xmlDoc->getElementsByTagName("TAPESERVER")->item($i)->getAttribute("MODIFDATE");
$TPCONFIG{$i}{'modifuser'} = $xmlDoc->getElementsByTagName("TAPESERVER")->item($i)->getAttribute("MODIFUSER");
$TPCONFIG{$i}{'tapedrivecomment'} = $xmlDoc->getElementsByTagName("TAPESERVER")->item($i)->getAttribute("TAPEDRIVECOMMENT");
$TPCONFIG{$i}{'tapeservicecomment'} = $xmlDoc->getElementsByTagName("TAPESERVER")->item($i)->getAttribute("TAPESERVICECOMMENT");
warn("$0: database entry nr. ".($i+1)." missing tape server hostname\n") unless ($TPCONFIG{$i}{'tapeserver'});
warn("$0: database entry nr. ".($i+1)." missing tape drive name\n") unless ($TPCONFIG{$i}{'tapedrive'});
warn("$0: database entry nr. ".($i+1)." missing device group name\n") unless ($TPCONFIG{$i}{'devicegroup'});
warn("$0: database entry nr. ".($i+1)." missing unix device\n") unless ($TPCONFIG{$i}{'unixdevice'});
warn("$0: database entry nr. ".($i+1)." missing init status\n") unless ($TPCONFIG{$i}{'initstatus'});
warn("$0: database entry nr. ".($i+1)." missing control method\n") unless ($TPCONFIG{$i}{'controlmethod'});
warn("$0: database entry nr. ".($i+1)." missing the modification date\n") unless ($TPCONFIG{$i}{'modifdate'});
warn("$0: database entry nr. ".($i+1)." missing user name\n") unless ($TPCONFIG{$i}{'modifuser'});
print("$0: database entry nr. ".($i+1)." no tape service comment\n") unless ($TPCONFIG{$i}{'tapeservicecomment'});
print("$0: database entry nr. ".($i+1)." no tape drive comment\n") unless ($TPCONFIG{$i}{'tapedrivecomment'});
}
$xmlDoc->dispose;
} else {
warn("$0: URL $url is not returning any usable data for $hostname. This tape server will not be configured. Please check whether there is a tape drive assigned to this tape server.\n");
}
} else {
warn("$0: URL $url doesn't seem to work. There could be a problem with the Web server or the Oracle APEX database server.\n");
}
return %TPCONFIG;
}
##########################################################################
sub UpdateFile {
##########################################################################
my ($filename, $newcontent) = @_;
my $changes = 0;
if ((-f $filename) and (-r $filename) and (-s $filename)) {
# Check the content of the file and correct it if there are some differences
$changes += LC::Check::file($filename,
source => $filename,
owner => 0,
group => 0,
mode => 0644,
backup => '.old',
code => sub {
my($oldcontent) = @_;
return() unless $oldcontent;
(my $oldfile = $oldcontent) =~ s/^#.*$//gim; # remove lines with comments
$oldfile =~ s/^\s*$//gim; # remove empty lines
(my $newfile = $newcontent) =~ s/^#.*$//gim; # remove lines with comments
$newfile =~ s/^\s*$//gim; # remove empty lines
$oldcontent = $newcontent unless ($oldfile eq $newfile);
return($oldcontent);
}
);
} else {
# The file is missing, create a new one
$changes += LC::File::file_contents($filename, $newcontent);
print "$0: created new $filename\n";
}
die ("$0: error modifying $filename\n") unless (defined($changes));
return $changes;
}
#!/usr/bin/python
"""
This script creates links to tape and medium changer devices.
The association between tape and smc device is made based on
the serial numbers stored in the TOMS DB.
"""
import re
import os
import sys
import socket
import pprint
import urllib2
import optparse
import cookielib
import subprocess
pp = pprint.PrettyPrinter(width=200)
#------------------------------------------------------------
def mklink(dev, sn, drivename, type):
if not options.noaction and drivename is not None:
link = '/dev/' + type + '_' + drivename
subprocess.Popen(['/bin/ln', '-f', '-s', dev, link]).wait()
print 'Created link', link
else:
print 'Cannot create link to ' + dev
print 'Drivename for serial number ' + sn + ' not found'
#------------------------------------------------------------
def fix_mismatch(mm_tape_dev, toms_drives, hostname):
#this fucntions assumes that there is only one mismatch,
#i.e. only one tape device with S/N not found in TOMS
#and only one drives in TOSM with S/N not found in the server.
l = []
cj = cookielib.CookieJar()
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
for d in toms_drives:
if d['match'] == 0:
#assigning drivename to the mismatched tape device
mm_tape_dev['drivename'] = d['drivename']
#fixing the S/N in TOMS
tomsurl = 'http://castortapeweb.cern.ch/cgi-bin/serial-nr-update.cgi?tapedrive=' + d['drivename'] + '&tapeserver=' + hostname + '&serialnumber=' + mm_tape_dev['sn']
if options.debug: print 'Opening', tomsurl
try:
urlfh = opener.open(tomsurl)
except:
print 'Cannot open ' + tomsurl
#------------------------------------------------------------
#main
#------------------------------------------------------------
# options ---------------------------------------------------
usage = "usage: %prog [options]"
parser = optparse.OptionParser(usage)
parser.add_option("-d", "--debug", action="store_true", dest="debug", help="print debug messages")
parser.add_option("--noaction", action="store_true", dest="noaction", help="do nothing")
(options, args) = parser.parse_args()
tape_devices = []
smc_devices = []
# find tape and smc devices
if options.debug: print 'Searching tape devices'
try:
p = subprocess.Popen(['/usr/bin/lsscsi', '-g'], stdout=subprocess.PIPE)
p.wait()
except:
print 'Cannot run lsscsi. Exit.'
sys.exit(0)
for line in p.stdout:
fields = line.split()
scsi_address = fields[0][1:-1]
type = fields[1]
scsi_generic = fields.pop()
scsi_tape = fields.pop()
if type == 'tape':
tape_devices.append({'scsi_address' : scsi_address, 'scsi_generic' : scsi_generic, 'scsi_tape' : scsi_tape, 'sn' : None, 'drivename' : None})
if type == 'mediumx':
smc_devices.append({'scsi_address' : scsi_address, 'scsi_generic' : scsi_generic, 'scsi_tape' : scsi_tape, 'sn' : None, 'drivename' : None})
ntpdev=len(tape_devices)
if options.debug:
print 'tape_devices:'
pp.pprint(tape_devices)
print 'smc_devices:'
pp.pprint(smc_devices)
# associate tape and smc devices
if options.debug: print 'Coupling tape and smc devices (if any)'
pairs = []
for tapedev in tape_devices:
for smcdev in smc_devices:
if tapedev['scsi_address'][:-2] == smcdev['scsi_address'][:-2]:
pairs.append([tapedev, smcdev])
if options.debug:
print 'pairs:'
pp.pprint(pairs)
if len(tape_devices)>len(smc_devices) and len(smc_devices)>0:
print 'Number of control paths lower that number of drives'
sys.exit(0)
# find the serial number of the tape device
# sg_inq will not work if the /dev/sgX device is not reachable
# sg_inq will not work if the /dev/nstY device is being used
# run sg_inq against the nst dev so that if it is already being used we exit
if options.debug: print 'Reading serial numbers from tape devices'
for tapedev in tape_devices:
tapedn = '/dev/nst'+tapedev['scsi_tape'][-1]
p = subprocess.Popen(['/usr/bin/sg_inq', tapedn], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if p.wait() != 0:
print 'Cannot run sg_inq on ' + tapedn + '. Exit'
sys.exit(0)
else:
for line in p.stdout:
#reg = re.search('(?<=Unit serial number: )(\d+)', line)
if re.search('Unit serial number', line):
l = line.split(':')
tapedev['sn'] = l[1][1:-1]
if tapedev['sn'] is None:
print 'Could not extract the serial number from the output of sg_inq ' + tapedn
sys.exit(0)
if options.debug:
print 'tape_devices:'
pp.pprint(tape_devices)
# search the drive names in toms by serial number
toms_drives = []
if options.debug: print 'Looking into TOMS for drive names'
hostname = socket.gethostname().split('.')[0]
tomsurl = 'https://apex.cern.ch/pls/htmldb_castorns/f?p=toms_prod:250:::NO::P250_TAPESERVER:' + hostname
if options.debug: print 'Opening', tomsurl
cj = cookielib.CookieJar()
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
try:
urlfh = opener.open(tomsurl)
except:
print 'Cannot open ' + tomsurl
sys.exit(0)
for line in urlfh:
if options.debug: print line
if not re.search('TAPESERVER', line): continue
drivename, serialnumber = '', ''
l = line.split()
for item in l:
g = item.split('=')
if g[0] == 'CURRENTSERIALNR': serialnumber = g[1][1:-1]
if g[0] == 'TAPEDRIVE': drivename = g[1][1:-1]
if drivename == '':
print 'drive name for host ' + hostname + ' not found in TOMS'
sys.exit(0)
if serialnumber == '':
print 'Serial number for drive', drivename, 'not found in TOMS'
#here we don't exit, in case of a signle mismatch we update the s/n
toms_drives.append({'drivename' : drivename, 'sn': serialnumber, 'match' : 0})
for tapedev in tape_devices:
if tapedev['sn'] == serialnumber:
tapedev['drivename'] = drivename
for d in toms_drives:
if d['drivename'] == drivename: d['match'] = 1
if options.debug:
print 'tape_devices:'
pp.pprint(tape_devices)
print 'toms_drives:'
pp.pprint(toms_drives)
#Check how many S/N are missing.
#1. If there is only one assume that the drive has been replaced and update the S/N in TOMS.
#2. If there are more than one the script does nothing (new or changed drives/devices
# will not be configured (link not created).
devs_mm_sn = 0
mm_tape_dev = None
for t in tape_devices:
if t['drivename'] is None:
devs_mm_sn += 1
mm_tape_dev = t
if devs_mm_sn == 1:
print 'One S/N mismatch. Going to fix S/N in TOMS'
fix_mismatch(mm_tape_dev, toms_drives, hostname)
elif devs_mm_sn == 0:
if options.debug: print 'No S/N mismatches'
else:
if options.debug: print 'Too many S/N mismatches'
# created links
if pairs == []:
# this is a SUN tape server
for tapedev in tape_devices:
tapedn = '/dev/nst'+tapedev['scsi_tape'][-1]
mklink(tapedn, tapedev['sn'], tapedev['drivename'], 'tape')
else:
#this is a IBM tape server
for pair in pairs:
tapedev = pair[0]
tapedn = '/dev/nst'+tapedev['scsi_tape'][-1]
mklink(tapedn, tapedev['sn'], tapedev['drivename'], 'tape')
smcdev = pair[1]
mklink(smcdev['scsi_generic'], tapedev['sn'], tapedev['drivename'], 'smc')
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment