Commit fb70163b authored by Michael Davis's avatar Michael Davis
Browse files

Merge remote-tracking branch 'origin/master' into migration_ci_tests

Conflicts:
	migration/castor/castorns_ctamigration_schema.sql
parents adad2d1c 2314a1cb
......@@ -3,6 +3,7 @@ logfile = /var/log/eos/fst/cta-fst-gcd.log
mgmhost = localhost.cern.ch
minfreebytes = 0
gcagesecs = 1
absolutemaxagesecs = 604800
queryperiodsecs = 20
mainloopperiodsecs = 10
xrdsecssskt = /etc/eos.keytab
......@@ -151,9 +151,6 @@ if [ "-${CI_CONTEXT}-" == '-systemd-' ]; then
systemctl status eos@{mq,mgm,fst} &>/dev/null && echo OK || echo FAILED
systemctl status eos@{mq,mgm,fst}
systemctl start cta-fst-gcd
else
# Using jemalloc as specified in
# it-puppet-module-eos:
......@@ -169,28 +166,6 @@ else
/usr/bin/xrootd -n mq -c /etc/xrd.cf.mq -l /var/log/eos/xrdlog.mq -b -Rdaemon
/usr/bin/xrootd -n mgm -c /etc/xrd.cf.mgm -m -l /var/log/eos/xrdlog.mgm -b -Rdaemon
/usr/bin/xrootd -n fst -c /etc/xrd.cf.fst -l /var/log/eos/xrdlog.fst -b -Rdaemon
runuser -u daemon setsid /usr/bin/cta-fst-gcd < /dev/null &
fi
echo "Giving cta-fst-gcd 3 second to start logging"
sleep 3
let EXPECTED_NB_STARTED_CTA_FST_GCD=NB_STARTED_CTA_FST_GCD+1
ACTUAL_NB_STARTED_CTA_FST_GCD=0
if test -f /var/log/eos/fst/cta-fst-gcd.log; then
ACTUAL_NB_STARTED_CTA_FST_GCD=`grep "cta-fst-gcd started" /var/log/eos/fst/cta-fst-gcd.log | wc -l`
else
echo "/var/log/eos/fst/cta-fst-gcd.log DOES NOT EXIST"
ps auxf | grep gcd
exit 1
fi
if test ${EXPECTED_NB_STARTED_CTA_FST_GCD} = ${ACTUAL_NB_STARTED_CTA_FST_GCD}; then
echo "/usr/bin/cta-fst-gcd LOGGED 'cta-fst-gcd started'"
else
echo "/usr/bin/cta-fst-gcd DID NOT LOG 'cta-fst-gcd started'"
exit 1
fi
eos vid enable krb5
......@@ -248,6 +223,22 @@ fi
sleep 1
done
# Start the FST garbage collector (the daemon user must be an EOS sudoer by now)
if [ "-${CI_CONTEXT}-" == '-systemd-' ]; then
systemctl start cta-fst-gcd
else
runuser -u daemon setsid /usr/bin/cta-fst-gcd > /dev/null 2>&1 < /dev/null &
fi
echo "Giving cta-fst-gcd 1 second to start"
sleep 1
FST_GCD_PID=`ps -ef | egrep '^daemon .* /bin/python /usr/bin/cta-fst-gcd$' | grep -v grep | awk '{print $2;}'`
if test "x${FST_GCD_PID}" = x; then
echo "cta-fst-gcd is not running"
exit 1
else
echo "cta-fst-gcd is running FST_GCD_PID=${FST_GCD_PID}"
fi
# test EOS
eos -b node ls
......
......@@ -60,19 +60,6 @@ CREATE TABLE CTADeltaDirsHelper (fileid INTEGER NOT NULL PRIMARY KEY, parent_fil
path VARCHAR2(2048), disk_uid INTEGER, disk_gid INTEGER, filemode INTEGER,
ctime INTEGER, mtime INTEGER, classid INTEGER);
CREATE GLOBAL TEMPORARY TABLE CTADeltaTemp(
fileid INTEGER NOT NULL PRIMARY KEY, parent_fileid INTEGER, depth INTEGER, path VARCHAR2(2048))
ON COMMIT PRESERVE ROWS;
CREATE TABLE CTADirsFailed(
fileid INTEGER NOT NULL PRIMARY KEY, depth INTEGER, path VARCHAR2(2048), disk_uid INTEGER, disk_gid INTEGER,
filemode INTEGER, ctime INTEGER, mtime INTEGER, classid INTEGER, retc INTEGER, message VARCHAR2(1024));
CREATE GLOBAL TEMPORARY TABLE CTADirsFailedTemp(
fileid INTEGER NOT NULL PRIMARY KEY, depth INTEGER, path VARCHAR2(2048), disk_uid INTEGER, disk_gid INTEGER,
filemode INTEGER, ctime INTEGER, mtime INTEGER, classid INTEGER, retc INTEGER, message VARCHAR2(1024))
ON COMMIT PRESERVE ROWS;
CREATE TABLE CTADirsFailed(
fileid INTEGER NOT NULL PRIMARY KEY, depth INTEGER, path VARCHAR2(2048), disk_uid INTEGER, disk_gid INTEGER,
filemode INTEGER, ctime INTEGER, mtime INTEGER, classid INTEGER, retc INTEGER, message VARCHAR2(1024));
......@@ -140,6 +127,9 @@ BEGIN
);
COMMIT;
SELECT COUNT(*) INTO nbFiles FROM CTAFilesHelper;
IF nbFiles = 0 THEN
raise_application_error(-20000, 'No valid files found on this tape pool, aborting the import');
END IF;
ctaLog(inPoolName, 'Intermediate table for files prepared, '|| nbFiles ||' files to be imported. ETA: '||
prettyTime(nbFiles/7000/60));
EXECUTE IMMEDIATE 'TRUNCATE TABLE CTADeltaDirsHelper';
......
......@@ -57,6 +57,7 @@ set -e
tapepool_castor_to_cta.py -t $1 -v $2 -i $3 $4
# execute the EOS metadata import
eos-import-dirs --delta # should there be any
eos-import-files
# terminate the export
......
......@@ -271,58 +271,73 @@ class Gc:
self.log.info("Config: mgmhost={}".format(self.config.mgmhost))
self.log.info("Config: minfreebytes={}".format(self.config.minfreebytes))
self.log.info("Config: gcagesecs={}".format(self.config.gcagesecs))
self.log.info("Config: absolutemaxagesecs={}".format(self.config.absolutemaxagesecs))
self.log.info("Config: queryperiodsecs={}". format(self.config.queryperiodsecs))
self.log.info("Config: mainloopperiodsecs={}". format(self.config.mainloopperiodsecs))
self.log.info("Config: xrdsecssskt={}".format(self.config.xrdsecssskt))
def processfile(self, subdir, fstfile):
fullpath = os.path.join(subdir,fstfile)
filesizeandctime = None
try:
filesizeandctime = self.disk.getfilesizeandctime(fullpath)
except Exception as err:
self.log.error(err)
if not filesizeandctime:
return
now = time.time()
agesecs = now - filesizeandctime.ctime
absolutemaxagereached = agesecs > self.config.absolutemaxagesecs
gcagereached = agesecs > self.config.gcagesecs
spacetracker = self.spacetrackers.gettracker(subdir)
totalfreebytes = spacetracker.getfreebytes()
shouldfreespace = totalfreebytes < self.config.minfreebytes
if shouldfreespace:
fullpath = os.path.join(subdir,fstfile)
filesizeandctime = None
if absolutemaxagereached or (shouldfreespace and gcagereached):
try:
filesizeandctime = self.disk.getfilesizeandctime(fullpath)
bytesrequiredbefore = 0
if self.config.minfreebytes > totalfreebytes:
bytesrequiredbefore = self.config.minfreebytes - totalfreebytes
self.eos.stagerrm(fstfile)
spacetracker.stagerrmqueued(filesizeandctime.sizebytes)
self.log.info("stagerrm: subdir={}, fxid={}, bytesrequiredbefore={}, filesizebytes={}, absolutemaxagereached={}, shouldfreespace={}, gcagereached={}"
.format(subdir, fstfile, bytesrequiredbefore, filesizeandctime.sizebytes, absolutemaxagereached, shouldfreespace, gcagereached))
nowstr = datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S.%f")
attrname = "sys.retrieve.error"
attrvalue = "Garbage collected at {}".format(nowstr)
self.eos.attrset(attrname, attrvalue, fstfile)
except StagerrmError as err:
pass
except Exception as err:
self.log.error(err)
if filesizeandctime:
now = time.time()
agesecs = now - filesizeandctime.ctime
if agesecs > self.config.gcagesecs:
try:
bytesrequiredbefore = self.config.minfreebytes - totalfreebytes
self.eos.stagerrm(fstfile)
spacetracker.stagerrmqueued(filesizeandctime.sizebytes)
self.log.info("stagerrm: subdir={}, fxid={}, bytesrequiredbefore={}, filesizebytes={}"
.format(subdir, fstfile, bytesrequiredbefore, filesizeandctime.sizebytes))
nowstr = datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S.%f")
attrname = "sys.retrieve.error"
attrvalue = "Garbage collected at {}".format(nowstr)
self.eos.attrset(attrname, attrvalue, fstfile)
except StagerrmError as err:
pass
except Exception as err:
self.log.error(err)
def processfssubdir(self, subdir):
spacetracker = self.spacetrackers.gettracker(subdir)
totalfreebytes = spacetracker.getfreebytes()
shouldfreespace = totalfreebytes < self.config.minfreebytes
if shouldfreespace:
subdirfiles = []
try:
subdirfiles = self.disk.listdir(subdir)
except Exception as err:
self.log.error("Failed to list contents of sub directory: subdir={}: {}".format(subdir, err))
#spacetracker = self.spacetrackers.gettracker(subdir)
#totalfreebytes = spacetracker.getfreebytes()
#shouldfreespace = totalfreebytes < self.config.minfreebytes
#if shouldfreespace:
# subdirfiles = []
# try:
# subdirfiles = self.disk.listdir(subdir)
# except Exception as err:
# self.log.error("Failed to list contents of sub directory: subdir={}: {}".format(subdir, err))
# fstfiles = [f for f in subdirfiles if re.match('^[0-9A-Fa-f]{8}$', f) and self.disk.isfile(os.path.join(subdir, f))]
# for fstfile in fstfiles:
# self.processfile(subdir, fstfile)
subdirfiles = []
try:
subdirfiles = self.disk.listdir(subdir)
except Exception as err:
self.log.error("Failed to list contents of sub directory: subdir={}: {}".format(subdir, err))
fstfiles = [f for f in subdirfiles if re.match('^[0-9A-Fa-f]{8}$', f) and self.disk.isfile(os.path.join(subdir, f))]
for fstfile in fstfiles:
self.processfile(subdir, fstfile)
fstfiles = [f for f in subdirfiles if re.match('^[0-9A-Fa-f]{8}$', f) and self.disk.isfile(os.path.join(subdir, f))]
for fstfile in fstfiles:
self.processfile(subdir, fstfile)
def processfs(self, path):
fsfiles = []
......@@ -424,6 +439,7 @@ def parseconf(conffile):
config.mgmhost = parser.get('main', 'mgmhost')
config.minfreebytes = parser.getint('main', 'minfreebytes')
config.gcagesecs = parser.getint('main', 'gcagesecs')
config.absolutemaxagesecs = parser.getint('main', 'absolutemaxagesecs')
config.queryperiodsecs = parser.getint('main', 'queryperiodsecs')
config.mainloopperiodsecs = parser.getint('main', 'mainloopperiodsecs')
config.xrdsecssskt = parser.get('main', 'xrdsecssskt')
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment