First attempt to create the utils RPM

operations/tape/cta-cern-tape-utils/Makefile

+SPECFILE = cta-cern-tape-utils.spec
+
+FILES = tape-config-generate
+
+rpmtopdir := $(shell rpm --eval %_topdir)
+rpmbuild  := $(shell [ -x /usr/bin/rpmbuild ] && echo rpmbuild || echo rpm)
+
+PKGNAME = $(shell grep -s '^Name:'    $(SPECFILE) | sed -e 's/Name: *//')
+PKGVERS = $(shell grep -s '^Version:' $(SPECFILE) | sed -e 's/Version: *//')
+
+rpm: $(SPECFILE)
+	tar cvfz $(rpmtopdir)/SOURCES/$(PKGNAME)-$(PKGVERS).tgz $(FILES)
+	cp -f $(SPECFILE) $(rpmtopdir)/SPECS/$(SPECFILE)
+	$(rpmbuild) -ba $(SPECFILE)

operations/tape/cta-cern-tape-utils/cta-cern-tape-utils.spec

+Summary: CTA CERN specific tape utilities
+Name: cta-cern-tape-utils
+Version: 1.0
+Release: 1
+License: GPL
+Buildroot: /tmp/%{name}-%{version}
+BuildArch: noarch
+Group: Application/CTA
+Requires: python36 cta-cli
+Source: %{name}-%{version}.tgz
+
+%description
+CTA CERN specific tape utilities:
+ - generate /etc/cta/TPCONFIG file from TOMS framework
+ - tape mount/unmount script
+ - tape label wrapper
+ - tape media check
+ - tape drive test
+
+Author: David Fernandez Alvarez, Vladimir Bahyl - 8/2019
+%prep
+
+%setup -c
+
+%install
+[ -d $RPM_BUILD_ROOT ] && rm -rf $RPM_BUILD_ROOT
+mkdir $RPM_BUILD_ROOT
+cd $RPM_BUILD_ROOT
+mkdir -p usr/local/bin etc/cron.d
+name=`echo %{name} |sed 's/CERN-CC-//'`
+
+install -m 755 $RPM_BUILD_DIR/%{name}-%{version}/tape-config-generate	usr/local/bin/tape-config-generate
+
+%post
+
+%clean
+rm -rf $RPM_BUILD_ROOT
+
+%files
+%defattr(-,root,root)
+/usr/local/bin/tape-config-generate

operations/tape/cta-cern-tape-utils/tape-config-generate

+#!/usr/bin/perl -w
+#######################################################################
+#
+# This script will generate /etc/cta/TPCONFIG file with the data from
+# TOMS (Tape Operations Management System) URL:
+#
+# https://apex.cern.ch/pls/htmldb_castorns/f?p=toms_prod:250:163672298908022::NO::P250_TAPESERVER:HOSTNAME
+#
+# Vladimir Bahyl - 05/2019
+#
+#######################################################################
+
+use strict;
+use XML::DOM;
+use Sys::Hostname;
+use LWP::UserAgent;
+use LC::Check qw(file);
+
+#use Data::Dumper;
+
+my $today = localtime;
+
+my %TPCONFIG = ();
+my $hostname = '';
+
+my $tpconfigfile = '/etc/castor/TPCONFIG';
+my $tpconfig = "#######################################################################
+#
+# CTA Tape Server Configuration file
+#
+# This tape server is not configured.
+#
+#######################################################################
+#
+# Generated on $today by $0
+";
+
+my $changes = 0;
+
+($hostname = hostname()) =~ s/\.cern\.ch$//io;
+
+my $configUrl = 'https://apex.cern.ch/pls/htmldb_castorns/f?p=toms_prod:250:163672298908022::NO::P250_TAPESERVER:HOSTNAME';
+die ("$0: missing configuration URL") unless ($configUrl);
+$configUrl =~ s/HOSTNAME/$hostname/o;
+
+#
+# Fetch the data
+#
+print("$0: Fetching the data over HTTP from the Oracle APEX database ... please be patient ...\n");
+%TPCONFIG = &GetData($configUrl);
+
+#
+# Prepare the TPCONFIG file
+#
+my $i = 0;
+while (%TPCONFIG and defined($TPCONFIG{$i}{'tapeserver'}) and (lc($TPCONFIG{$i}{'tapeserver'}) eq lc($hostname))) {
+  $tpconfig = "#######################################################################
+#
+# CTA Tape Server Configuration file
+#
+# unit      device    system                control
+# name      group     device                method
+
+" if ($i == 0);
+
+  $tpconfig .= "$TPCONFIG{$i}{'tapedrive'}    $TPCONFIG{$i}{'devicegroup'}    $TPCONFIG{$i}{'unixdevice'}    $TPCONFIG{$i}{'controlmethod'}
+
+# Tape Drive Comment: $TPCONFIG{$i}{'tapedrivecomment'}
+# Tape Service Comment: $TPCONFIG{$i}{'tapeservicecomment'}
+# Modified by: $TPCONFIG{$i}{'modifuser'}
+# Modify date: $TPCONFIG{$i}{'modifdate'}
+
+";
+  $i++;
+}
+
+$tpconfig .= "#
+#######################################################################
+#
+# Generated on $today by $0
+" if (%TPCONFIG and (lc($TPCONFIG{0}{'tapeserver'}) eq lc($hostname)));
+
+# Change the TPCONFIG location if comment mentions CTA
+$tpconfigfile = '/etc/cta/TPCONFIG' if (%TPCONFIG and (defined $TPCONFIG{0}{'tapeservicecomment'}) and ($TPCONFIG{0}{'tapeservicecomment'} =~ /CTA/oi));
+
+#
+# Configure TPCONFIG and SSI files
+#
+$changes += &UpdateFile($tpconfigfile, $tpconfig);
+
+LC::Check::link('/etc/TPCONFIG', $tpconfigfile,
+  backup  => '.old',
+  nocheck => 1,
+  force   => 1
+);
+
+##########################################################################
+sub GetData {
+##########################################################################
+  my ($url) = @_;
+
+  my %TPCONFIG = ();
+
+  my $xmlParser = new XML::DOM::Parser;
+
+  # Download the XML formated configuration data
+  # Use cookies because of APEX (otherwise, nothing will be downloaded; redirection will not work)
+  my $UserAgent = LWP::UserAgent->new;
+  $UserAgent->cookie_jar({ file => undef });
+  my $xml = $UserAgent->get($url);
+
+  if (defined ($xml->content)) {
+    if ($xml->content =~/TAPESERVER/oi) {
+      my $xmlDoc = $xmlParser->parse($xml->content);
+
+      # pcitpdp39 ~ > lynx -source "http://oraweb.cern.ch/pls/cdbsqldev/web.show_tpconfig?p_tapeserver=tpsrv027&p_output=xml"
+      # <?xml version = '1.0'?>
+      # <TPCONFIGSEARCHLIST> <TAPESERVER NAME="tpsrv027" TAPEDRIVE="994B53A6" DEVICEGROUP="994BR5" UNIXDEVICE="/dev/nst0" DENSITY="200G" COMPRESSION="Y" INITSTATUS="DOWN" CONTROLMETHOD="acs0,3,10,6" MODEL="9940" ROBOTHOST="sunstk62" /> </TPCONFIGSEARCHLIST>
+
+      for my $i (0 .. ($xmlDoc->getElementsByTagName("TAPESERVER")->getLength())-1) {
+        $TPCONFIG{$i}{'tapeserver'}         = $xmlDoc->getElementsByTagName("TAPESERVER")->item($i)->getAttribute("NAME");
+        $TPCONFIG{$i}{'tapedrive'}          = $xmlDoc->getElementsByTagName("TAPESERVER")->item($i)->getAttribute("TAPEDRIVE");
+        $TPCONFIG{$i}{'devicegroup'}        = $xmlDoc->getElementsByTagName("TAPESERVER")->item($i)->getAttribute("DEVICEGROUP");
+        $TPCONFIG{$i}{'unixdevice'}         = $xmlDoc->getElementsByTagName("TAPESERVER")->item($i)->getAttribute("UNIXDEVICE");
+        $TPCONFIG{$i}{'initstatus'}         = lc($xmlDoc->getElementsByTagName("TAPESERVER")->item($i)->getAttribute("INITSTATUS"));
+        $TPCONFIG{$i}{'controlmethod'}      = $xmlDoc->getElementsByTagName("TAPESERVER")->item($i)->getAttribute("CONTROLMETHOD");
+        $TPCONFIG{$i}{'modifdate'}          = $xmlDoc->getElementsByTagName("TAPESERVER")->item($i)->getAttribute("MODIFDATE");
+        $TPCONFIG{$i}{'modifuser'}          = $xmlDoc->getElementsByTagName("TAPESERVER")->item($i)->getAttribute("MODIFUSER");
+        $TPCONFIG{$i}{'tapedrivecomment'}   = $xmlDoc->getElementsByTagName("TAPESERVER")->item($i)->getAttribute("TAPEDRIVECOMMENT");
+        $TPCONFIG{$i}{'tapeservicecomment'} = $xmlDoc->getElementsByTagName("TAPESERVER")->item($i)->getAttribute("TAPESERVICECOMMENT");
+
+        warn("$0: database entry nr. ".($i+1)." missing tape server hostname\n")    unless ($TPCONFIG{$i}{'tapeserver'});
+        warn("$0: database entry nr. ".($i+1)." missing tape drive name\n")         unless ($TPCONFIG{$i}{'tapedrive'});
+        warn("$0: database entry nr. ".($i+1)." missing device group name\n")       unless ($TPCONFIG{$i}{'devicegroup'});
+        warn("$0: database entry nr. ".($i+1)." missing unix device\n")             unless ($TPCONFIG{$i}{'unixdevice'});
+        warn("$0: database entry nr. ".($i+1)." missing init status\n")             unless ($TPCONFIG{$i}{'initstatus'});
+        warn("$0: database entry nr. ".($i+1)." missing control method\n")          unless ($TPCONFIG{$i}{'controlmethod'});
+        warn("$0: database entry nr. ".($i+1)." missing the modification date\n")   unless ($TPCONFIG{$i}{'modifdate'});
+        warn("$0: database entry nr. ".($i+1)." missing user name\n")               unless ($TPCONFIG{$i}{'modifuser'});
+        print("$0: database entry nr. ".($i+1)." no tape service comment\n")        unless ($TPCONFIG{$i}{'tapeservicecomment'});
+        print("$0: database entry nr. ".($i+1)." no tape drive comment\n")          unless ($TPCONFIG{$i}{'tapedrivecomment'});
+      }
+
+      $xmlDoc->dispose;
+    } else {
+      warn("$0: URL $url is not returning any usable data for $hostname. This tape server will not be configured. Please check whether there is a tape drive assigned to this tape server.\n");
+    }
+  } else {
+    warn("$0: URL $url doesn't seem to work. There could be a problem with the Web server or the Oracle APEX database server.\n");
+  }
+
+  return %TPCONFIG;
+}
+
+##########################################################################
+sub UpdateFile {
+##########################################################################
+  my ($filename, $newcontent) = @_;
+
+  my $changes = 0;
+
+  if ((-f $filename) and (-r $filename) and (-s $filename)) {
+      # Check the content of the file and correct it if there are some differences
+      $changes += LC::Check::file($filename,
+      source => $filename,
+      owner  => 0,
+      group  => 0,
+      mode   => 0644,
+      backup => '.old',
+      code   => sub {
+        my($oldcontent) = @_;
+        return() unless $oldcontent;
+
+        (my $oldfile = $oldcontent) =~ s/^#.*$//gim; # remove lines with comments
+        $oldfile =~ s/^\s*$//gim;                    # remove empty lines
+
+        (my $newfile = $newcontent) =~ s/^#.*$//gim; # remove lines with comments
+        $newfile =~ s/^\s*$//gim;                    # remove empty lines
+
+        $oldcontent = $newcontent unless ($oldfile eq $newfile);
+
+        return($oldcontent);
+      }
+    );
+  } else {
+    # The file is missing, create a new one
+    $changes += LC::File::file_contents($filename, $newcontent);
+    print "$0: created new $filename\n";
+  }
+  die ("$0: error modifying $filename\n") unless (defined($changes));
+
+  return $changes;
+}

operations/tape/cta-cern-tape-utils/tape-devices-namer

+#!/usr/bin/python
+
+"""
+This script creates links to tape and medium changer devices.
+The association between tape and smc device is made based on
+the serial numbers stored in the TOMS DB.
+"""
+
+import re
+import os
+import sys
+import socket 
+import pprint
+import urllib2
+import optparse
+import cookielib
+import subprocess 
+pp = pprint.PrettyPrinter(width=200)
+
+#------------------------------------------------------------
+def mklink(dev, sn, drivename, type):
+    if not options.noaction and drivename is not None:
+        link = '/dev/' + type + '_' + drivename
+        subprocess.Popen(['/bin/ln', '-f',  '-s', dev, link]).wait()
+        print 'Created link', link
+    else:
+        print 'Cannot create link to ' + dev
+        print 'Drivename for serial number ' + sn + ' not found'
+
+#------------------------------------------------------------
+def fix_mismatch(mm_tape_dev, toms_drives, hostname):
+    #this fucntions assumes that there is only one mismatch,
+    #i.e. only one tape device with S/N not found in TOMS
+    #and only one drives in TOSM with S/N not found in the server.
+    l = []
+    cj = cookielib.CookieJar()
+    opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
+    for d in toms_drives:
+        if d['match'] == 0:
+            #assigning drivename to the mismatched tape device 
+            mm_tape_dev['drivename'] = d['drivename']
+            #fixing the S/N in TOMS
+            tomsurl = 'http://castortapeweb.cern.ch/cgi-bin/serial-nr-update.cgi?tapedrive=' + d['drivename'] + '&tapeserver=' + hostname + '&serialnumber=' + mm_tape_dev['sn']
+            if options.debug: print 'Opening', tomsurl
+            try:
+                urlfh = opener.open(tomsurl)
+            except:
+                print 'Cannot open ' + tomsurl
+                
+            
+#------------------------------------------------------------
+#main
+#------------------------------------------------------------
+
+# options ---------------------------------------------------
+usage = "usage: %prog [options]"
+parser = optparse.OptionParser(usage)
+parser.add_option("-d", "--debug", action="store_true", dest="debug", help="print debug messages")
+parser.add_option("--noaction", action="store_true", dest="noaction", help="do nothing")
+(options, args) = parser.parse_args()
+
+tape_devices = []
+smc_devices = []
+
+# find tape and smc devices
+if options.debug: print 'Searching tape devices'
+try:
+    p = subprocess.Popen(['/usr/bin/lsscsi', '-g'], stdout=subprocess.PIPE)
+    p.wait()
+except:
+    print 'Cannot run lsscsi. Exit.'
+    sys.exit(0)
+    
+for line in p.stdout:
+    fields = line.split() 
+    scsi_address = fields[0][1:-1]
+    type = fields[1]
+    scsi_generic = fields.pop()
+    scsi_tape = fields.pop()
+    if type == 'tape':
+        tape_devices.append({'scsi_address' : scsi_address, 'scsi_generic' : scsi_generic, 'scsi_tape' : scsi_tape, 'sn' : None, 'drivename' : None})
+    if type == 'mediumx':
+        smc_devices.append({'scsi_address' : scsi_address, 'scsi_generic' : scsi_generic, 'scsi_tape' : scsi_tape, 'sn' : None, 'drivename' : None})
+
+ntpdev=len(tape_devices)
+
+if options.debug:
+    print 'tape_devices:'
+    pp.pprint(tape_devices)
+    print 'smc_devices:'
+    pp.pprint(smc_devices)
+
+# associate tape and smc devices
+if options.debug: print 'Coupling tape and smc devices (if any)'
+pairs = []
+for tapedev in tape_devices:
+    for smcdev in smc_devices:
+        if tapedev['scsi_address'][:-2] == smcdev['scsi_address'][:-2]:
+            pairs.append([tapedev, smcdev])
+if options.debug:
+    print 'pairs:'
+    pp.pprint(pairs)
+
+if len(tape_devices)>len(smc_devices) and len(smc_devices)>0:
+    print 'Number of control paths lower that number of drives'
+    sys.exit(0)
+
+
+# find the serial number of the tape device
+# sg_inq will not work if the /dev/sgX device is not reachable
+# sg_inq will not work if the /dev/nstY device is being used
+# run sg_inq against the nst dev so that if it is already being used we exit
+if options.debug: print 'Reading serial numbers from tape devices'
+
+for tapedev in tape_devices:
+    tapedn = '/dev/nst'+tapedev['scsi_tape'][-1]
+    p = subprocess.Popen(['/usr/bin/sg_inq', tapedn], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    if p.wait() != 0:
+        print 'Cannot run sg_inq on ' +  tapedn + '. Exit'
+        sys.exit(0)
+    else:
+        for line in p.stdout:
+            #reg = re.search('(?<=Unit serial number: )(\d+)', line)
+            if re.search('Unit serial number', line):
+                l = line.split(':')
+                tapedev['sn'] = l[1][1:-1]
+
+    if tapedev['sn'] is None:
+        print 'Could not extract the serial number from the output of sg_inq ' + tapedn
+        sys.exit(0)
+                
+if options.debug:
+    print 'tape_devices:'
+    pp.pprint(tape_devices)
+
+
+# search the drive names in toms by serial number
+toms_drives = []
+if options.debug: print 'Looking into TOMS for drive names'
+hostname = socket.gethostname().split('.')[0]
+tomsurl =  'https://apex.cern.ch/pls/htmldb_castorns/f?p=toms_prod:250:::NO::P250_TAPESERVER:' + hostname
+if options.debug: print 'Opening', tomsurl
+
+cj = cookielib.CookieJar()
+opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
+
+try:
+    urlfh = opener.open(tomsurl)
+except:
+    print 'Cannot open ' + tomsurl
+    sys.exit(0)
+
+for line in urlfh:
+
+    if options.debug: print line
+
+    if not re.search('TAPESERVER', line): continue
+
+    drivename, serialnumber = '', ''
+
+    l = line.split()
+    for item in l:
+        g = item.split('=')
+        if g[0] == 'CURRENTSERIALNR': serialnumber =  g[1][1:-1]
+        if g[0] == 'TAPEDRIVE': drivename =  g[1][1:-1]
+
+    if drivename == '':
+        print 'drive name for host ' + hostname + ' not found in TOMS'
+        sys.exit(0)
+
+    if serialnumber  == '':
+        print 'Serial number for drive', drivename, 'not found in TOMS'
+        #here we don't exit, in case of a signle mismatch we update the s/n
+
+    toms_drives.append({'drivename' : drivename, 'sn': serialnumber, 'match' : 0})
+        
+    for tapedev in tape_devices:
+        if tapedev['sn'] == serialnumber:
+            tapedev['drivename'] = drivename
+            for d in toms_drives:
+                if d['drivename'] == drivename: d['match'] = 1
+
+if options.debug:
+    print 'tape_devices:'
+    pp.pprint(tape_devices)
+    print 'toms_drives:'
+    pp.pprint(toms_drives)
+
+
+#Check how many S/N are missing.
+#1. If there is only one assume that the drive has been replaced and update the S/N in TOMS.
+#2. If there are more than one the script does nothing (new or changed drives/devices
+#   will not be configured (link not created).
+
+devs_mm_sn = 0
+mm_tape_dev = None
+for t in tape_devices:
+    if t['drivename'] is None:
+        devs_mm_sn += 1
+        mm_tape_dev = t
+
+if devs_mm_sn == 1:
+    print 'One S/N mismatch. Going to fix S/N in TOMS'
+    fix_mismatch(mm_tape_dev, toms_drives, hostname)
+elif devs_mm_sn == 0:
+    if options.debug: print 'No S/N mismatches'
+else:
+    if options.debug: print 'Too many S/N mismatches'
+    
+
+# created links
+if pairs == []:
+    # this is a SUN tape server
+    for tapedev in tape_devices:
+        tapedn = '/dev/nst'+tapedev['scsi_tape'][-1]
+        mklink(tapedn, tapedev['sn'], tapedev['drivename'], 'tape')
+else:
+    #this is a IBM tape server
+    for pair in pairs:
+        tapedev = pair[0]
+        tapedn = '/dev/nst'+tapedev['scsi_tape'][-1]
+        mklink(tapedn, tapedev['sn'], tapedev['drivename'], 'tape')
+        smcdev = pair[1]
+        mklink(smcdev['scsi_generic'], tapedev['sn'], tapedev['drivename'], 'smc')
+
+
+