Merge remote-tracking branch 'origin/master' into migration_ci_tests

Conflicts:
	migration/castor/castorns_ctamigration_schema.sql

continuousintegration/docker/ctafrontend/cc7/config/ctaeos/etc/cta/cta-fst-gcd.conf

@@ -3,6 +3,7 @@ logfile = /var/log/eos/fst/cta-fst-gcd.log
 mgmhost = localhost.cern.ch
 minfreebytes = 0
 gcagesecs = 1
+absolutemaxagesecs = 604800
 queryperiodsecs = 20
 mainloopperiodsecs = 10
 xrdsecssskt = /etc/eos.keytab

continuousintegration/docker/ctafrontend/cc7/opt/run/bin/ctaeos-mgm.sh

@@ -151,9 +151,6 @@ if [ "-${CI_CONTEXT}-" == '-systemd-' ]; then
   systemctl status eos@{mq,mgm,fst} &>/dev/null && echo OK || echo FAILED
 
   systemctl status eos@{mq,mgm,fst}
-
-  systemctl start cta-fst-gcd
-
 else
   # Using jemalloc as specified in
   # it-puppet-module-eos:
@@ -169,28 +166,6 @@ else
     /usr/bin/xrootd -n mq -c /etc/xrd.cf.mq -l /var/log/eos/xrdlog.mq -b -Rdaemon
     /usr/bin/xrootd -n mgm -c /etc/xrd.cf.mgm -m -l /var/log/eos/xrdlog.mgm -b -Rdaemon
     /usr/bin/xrootd -n fst -c /etc/xrd.cf.fst -l /var/log/eos/xrdlog.fst -b -Rdaemon
-
-
-  runuser -u daemon setsid /usr/bin/cta-fst-gcd < /dev/null &
-fi
-
-echo "Giving cta-fst-gcd 3 second to start logging"
-sleep 3
-
-let EXPECTED_NB_STARTED_CTA_FST_GCD=NB_STARTED_CTA_FST_GCD+1
-ACTUAL_NB_STARTED_CTA_FST_GCD=0
-if test -f /var/log/eos/fst/cta-fst-gcd.log; then
-  ACTUAL_NB_STARTED_CTA_FST_GCD=`grep "cta-fst-gcd started" /var/log/eos/fst/cta-fst-gcd.log | wc -l`
-else
-  echo "/var/log/eos/fst/cta-fst-gcd.log DOES NOT EXIST"
-  ps auxf | grep gcd
-  exit 1
-fi
-if test ${EXPECTED_NB_STARTED_CTA_FST_GCD} = ${ACTUAL_NB_STARTED_CTA_FST_GCD}; then
-  echo "/usr/bin/cta-fst-gcd LOGGED 'cta-fst-gcd started'"
-else
-  echo "/usr/bin/cta-fst-gcd DID NOT LOG 'cta-fst-gcd started'"
-  exit 1
 fi
 
   eos vid enable krb5
@@ -248,6 +223,22 @@ fi
     sleep 1
   done
 
+  # Start the FST garbage collector (the daemon user must be an EOS sudoer by now)
+  if [ "-${CI_CONTEXT}-" == '-systemd-' ]; then
+    systemctl start cta-fst-gcd
+  else
+    runuser -u daemon setsid /usr/bin/cta-fst-gcd > /dev/null 2>&1 < /dev/null &
+  fi
+  echo "Giving cta-fst-gcd 1 second to start"
+  sleep 1
+  FST_GCD_PID=`ps -ef | egrep '^daemon .* /bin/python /usr/bin/cta-fst-gcd$' | grep -v grep | awk '{print $2;}'`
+  if test "x${FST_GCD_PID}" = x; then
+    echo "cta-fst-gcd is not running"
+    exit 1
+  else
+    echo "cta-fst-gcd is running FST_GCD_PID=${FST_GCD_PID}"
+  fi
+
 # test EOS
   eos -b node ls
 

python/eosfstgcd/ctafstgcd.py

@@ -271,58 +271,73 @@ class Gc:
     self.log.info("Config: mgmhost={}".format(self.config.mgmhost))
     self.log.info("Config: minfreebytes={}".format(self.config.minfreebytes))
     self.log.info("Config: gcagesecs={}".format(self.config.gcagesecs))
+    self.log.info("Config: absolutemaxagesecs={}".format(self.config.absolutemaxagesecs))
     self.log.info("Config: queryperiodsecs={}". format(self.config.queryperiodsecs))
     self.log.info("Config: mainloopperiodsecs={}". format(self.config.mainloopperiodsecs))
     self.log.info("Config: xrdsecssskt={}".format(self.config.xrdsecssskt))
 
   def processfile(self, subdir, fstfile):
+    fullpath = os.path.join(subdir,fstfile)
+    filesizeandctime = None
+    try:
+      filesizeandctime = self.disk.getfilesizeandctime(fullpath)
+    except Exception as err:
+      self.log.error(err)
+
+    if not filesizeandctime:
+      return
+
+    now = time.time()
+    agesecs = now - filesizeandctime.ctime
+    absolutemaxagereached = agesecs > self.config.absolutemaxagesecs
+    gcagereached = agesecs > self.config.gcagesecs
     spacetracker = self.spacetrackers.gettracker(subdir)
     totalfreebytes = spacetracker.getfreebytes()
     shouldfreespace = totalfreebytes < self.config.minfreebytes
 
-    if shouldfreespace:
-      fullpath = os.path.join(subdir,fstfile)
-
-      filesizeandctime = None
+    if absolutemaxagereached or (shouldfreespace and gcagereached):
       try:
-        filesizeandctime = self.disk.getfilesizeandctime(fullpath)
+        bytesrequiredbefore = 0
+        if self.config.minfreebytes > totalfreebytes:
+          bytesrequiredbefore = self.config.minfreebytes - totalfreebytes
+        self.eos.stagerrm(fstfile)
+        spacetracker.stagerrmqueued(filesizeandctime.sizebytes)
+        self.log.info("stagerrm: subdir={}, fxid={}, bytesrequiredbefore={}, filesizebytes={}, absolutemaxagereached={}, shouldfreespace={}, gcagereached={}"
+          .format(subdir, fstfile, bytesrequiredbefore, filesizeandctime.sizebytes, absolutemaxagereached, shouldfreespace, gcagereached))
+        nowstr = datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S.%f")
+        attrname = "sys.retrieve.error"
+        attrvalue = "Garbage collected at {}".format(nowstr)
+        self.eos.attrset(attrname, attrvalue, fstfile)
+      except StagerrmError as err:
+        pass
       except Exception as err:
         self.log.error(err)
 
-      if filesizeandctime:
-        now = time.time()
-        agesecs = now - filesizeandctime.ctime
-        if agesecs > self.config.gcagesecs:
-          try:
-            bytesrequiredbefore = self.config.minfreebytes - totalfreebytes
-            self.eos.stagerrm(fstfile)
-            spacetracker.stagerrmqueued(filesizeandctime.sizebytes)
-            self.log.info("stagerrm: subdir={}, fxid={}, bytesrequiredbefore={}, filesizebytes={}"
-              .format(subdir, fstfile, bytesrequiredbefore, filesizeandctime.sizebytes))
-            nowstr = datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S.%f")
-            attrname = "sys.retrieve.error"
-            attrvalue = "Garbage collected at {}".format(nowstr)
-            self.eos.attrset(attrname, attrvalue, fstfile)
-          except StagerrmError as err:
-            pass
-          except Exception as err:
-            self.log.error(err)
-
   def processfssubdir(self, subdir):
-    spacetracker = self.spacetrackers.gettracker(subdir)
-    totalfreebytes = spacetracker.getfreebytes()
-    shouldfreespace = totalfreebytes < self.config.minfreebytes
-
-    if shouldfreespace:
-      subdirfiles = []
-      try:
-        subdirfiles = self.disk.listdir(subdir)
-      except Exception as err:
-        self.log.error("Failed to list contents of sub directory: subdir={}: {}".format(subdir, err))
+    #spacetracker = self.spacetrackers.gettracker(subdir)
+    #totalfreebytes = spacetracker.getfreebytes()
+    #shouldfreespace = totalfreebytes < self.config.minfreebytes
+
+    #if shouldfreespace:
+    #  subdirfiles = []
+    #  try:
+    #    subdirfiles = self.disk.listdir(subdir)
+    #  except Exception as err:
+    #    self.log.error("Failed to list contents of sub directory: subdir={}: {}".format(subdir, err))
+
+    #  fstfiles = [f for f in subdirfiles if re.match('^[0-9A-Fa-f]{8}$', f) and self.disk.isfile(os.path.join(subdir, f))]
+    #  for fstfile in fstfiles:
+    #    self.processfile(subdir, fstfile)
+
+    subdirfiles = []
+    try:
+      subdirfiles = self.disk.listdir(subdir)
+    except Exception as err:
+      self.log.error("Failed to list contents of sub directory: subdir={}: {}".format(subdir, err))
 
-      fstfiles = [f for f in subdirfiles if re.match('^[0-9A-Fa-f]{8}$', f) and self.disk.isfile(os.path.join(subdir, f))]
-      for fstfile in fstfiles:
-        self.processfile(subdir, fstfile)
+    fstfiles = [f for f in subdirfiles if re.match('^[0-9A-Fa-f]{8}$', f) and self.disk.isfile(os.path.join(subdir, f))]
+    for fstfile in fstfiles:
+      self.processfile(subdir, fstfile)
 
   def processfs(self, path):
     fsfiles = []
@@ -424,6 +439,7 @@ def parseconf(conffile):
     config.mgmhost = parser.get('main', 'mgmhost')
     config.minfreebytes = parser.getint('main', 'minfreebytes')
     config.gcagesecs = parser.getint('main', 'gcagesecs')
+    config.absolutemaxagesecs = parser.getint('main', 'absolutemaxagesecs')
     config.queryperiodsecs = parser.getint('main', 'queryperiodsecs')
     config.mainloopperiodsecs = parser.getint('main', 'mainloopperiodsecs')
     config.xrdsecssskt = parser.get('main', 'xrdsecssskt')