Skip to content
Snippets Groups Projects
Commit 87486e45 authored by Steven Murray's avatar Steven Murray
Browse files

cta/CTA#618 cta-fst-gcd now sets sys.retrieve.error when a file has been garbage collected 

An example value of sys.retrieve.error is as follows:

[itctabuild02] ~ > sudo eos attr ls /eos/dev/userfiles/testdir_1/test_1 | grep sys.retrieve.error
sys.retrieve.error="Garbage collected at 2019/09/09 16:27:26.609570"
[itctabuild02] ~ >
parent a0b7e948
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
python/eosfstgcd/ctafstgcd.py
+ 23
− 0
View file @ 87486e45
......@@ -37,6 +37,9 @@ class UserError(Exception):
class StagerrmError(Exception):
pass
class AttrsetError(Exception):
pass
class NoMgmHost(UserError):
pass
......@@ -142,6 +145,22 @@ class RealEos:
if 0 != process.returncode:
raise StagerrmError("'{}' returned non zero: returncode={}".format(cmd, process.returncode))
def attrset(self, name, value, fxid):
mgmurl = "root://{}".format(self.mgmhost)
args = ["eos", "-r", "0", "0", mgmurl, "attr", "set", '{}={}'.format(name, value), "fxid:{}".format(fxid)]
env = os.environ.copy()
env["XrdSecPROTOCOL"] = "sss"
env["XrdSecSSSKT"] = self.xrdsecssskt
process = None
try:
process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
except Exception as err:
raise Exception("Failed to execute '{}': {}".format(" ".join(args), err))
stdout,stderr = process.communicate()
if 0 != process.returncode:
raise AttrsetError("'{}' returned non zero: returncode={}".format(" ".join(args), process.returncode))
class SpaceTracker:
'''Calculates the amount of effective free space in the file system of a given
file or directory by querying the OS and taking into account the pending
......@@ -280,6 +299,10 @@ class Gc:
spacetracker.stagerrmqueued(filesizeandctime.sizebytes)
self.log.info("stagerrm: subdir={}, fxid={}, bytesrequiredbefore={}, filesizebytes={}"
.format(subdir, fstfile, bytesrequiredbefore, filesizeandctime.sizebytes))
nowstr = datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S.%f")
attrname = "sys.retrieve.error"
attrvalue = "Garbage collected at {}".format(nowstr)
self.eos.attrset(attrname, attrvalue, fstfile)
except StagerrmError as err:
pass
except Exception as err:
......
python/eosfstgcd/test_ctafstgcd.py
+ 30
− 0
View file @ 87486e45
......@@ -113,6 +113,7 @@ class MockEos:
self.filesystems = filesystems
self.nbfsls = 0
self.nbstagerrm = 0
self.nbattrset = 0
def fsls(self):
self.nbfsls = self.nbfsls + 1
......@@ -121,6 +122,9 @@ class MockEos:
def stagerrm(self, fxid):
self.nbstagerrm = self.nbstagerrm + 1
def attrset(self, name, value, fxid):
self.nbattrset = self.nbattrset + 1
class RealDiskCase(unittest.TestCase):
def setUp(self):
self.log = DummyLogger()
......@@ -234,6 +238,7 @@ class GcTestCase(unittest.TestCase):
self.assertEqual(0, disk.nbgetfilesizeandctime)
self.assertEqual(0, eos.nbfsls)
self.assertEqual(0, eos.nbstagerrm)
self.assertEqual(0, eos.nbattrset)
gc = ctafstgcd.Gc(self.log, self.fqdn, disk, eos, self.config)
......@@ -244,6 +249,7 @@ class GcTestCase(unittest.TestCase):
self.assertEqual(0, disk.nbgetfilesizeandctime)
self.assertEqual(0, eos.nbfsls)
self.assertEqual(0, eos.nbstagerrm)
self.assertEqual(0, eos.nbattrset)
def test_run_only_once_no_fs(self):
disk = MockDisk(self.mocktree, self.freebytes, self.filesizeandctime)
......@@ -256,6 +262,7 @@ class GcTestCase(unittest.TestCase):
self.assertEqual(0, disk.nbgetfilesizeandctime)
self.assertEqual(0, eos.nbfsls)
self.assertEqual(0, eos.nbstagerrm)
self.assertEqual(0, eos.nbattrset)
gc = ctafstgcd.Gc(self.log, self.fqdn, disk, eos, self.config)
......@@ -266,6 +273,7 @@ class GcTestCase(unittest.TestCase):
self.assertEqual(0, disk.nbgetfilesizeandctime)
self.assertEqual(0, eos.nbfsls)
self.assertEqual(0, eos.nbstagerrm)
self.assertEqual(0, eos.nbattrset)
runonlyonce = True
gc.run(runonlyonce)
......@@ -277,6 +285,7 @@ class GcTestCase(unittest.TestCase):
self.assertEqual(0, disk.nbgetfilesizeandctime)
self.assertEqual(1, eos.nbfsls)
self.assertEqual(0, eos.nbstagerrm)
self.assertEqual(0, eos.nbattrset)
def test_run_only_once_one_fs(self):
mockfs = MockTreeNode("filesystem1")
......@@ -297,6 +306,7 @@ class GcTestCase(unittest.TestCase):
self.assertEqual(0, disk.nbgetfilesizeandctime)
self.assertEqual(0, eos.nbfsls)
self.assertEqual(0, eos.nbstagerrm)
self.assertEqual(0, eos.nbattrset)
gc = ctafstgcd.Gc(self.log, self.fqdn, disk, eos, self.config)
......@@ -307,6 +317,7 @@ class GcTestCase(unittest.TestCase):
self.assertEqual(0, disk.nbgetfilesizeandctime)
self.assertEqual(0, eos.nbfsls)
self.assertEqual(0, eos.nbstagerrm)
self.assertEqual(0, eos.nbattrset)
runonlyonce = True
gc.run(runonlyonce)
......@@ -318,6 +329,7 @@ class GcTestCase(unittest.TestCase):
self.assertEqual(0, disk.nbgetfilesizeandctime)
self.assertEqual(1, eos.nbfsls)
self.assertEqual(0, eos.nbstagerrm)
self.assertEqual(0, eos.nbattrset)
def test_run_only_once_one_fs_one_subdir_no_free_space(self):
mocksubdir = MockTreeNode("12345678")
......@@ -339,6 +351,7 @@ class GcTestCase(unittest.TestCase):
self.assertEqual(0, disk.nbgetfilesizeandctime)
self.assertEqual(0, eos.nbfsls)
self.assertEqual(0, eos.nbstagerrm)
self.assertEqual(0, eos.nbattrset)
self.config.minfreebytes = self.freebytes + 1
......@@ -351,6 +364,7 @@ class GcTestCase(unittest.TestCase):
self.assertEqual(0, disk.nbgetfilesizeandctime)
self.assertEqual(0, eos.nbfsls)
self.assertEqual(0, eos.nbstagerrm)
self.assertEqual(0, eos.nbattrset)
runonlyonce = True
gc.run(runonlyonce)
......@@ -362,6 +376,7 @@ class GcTestCase(unittest.TestCase):
self.assertEqual(0, disk.nbgetfilesizeandctime)
self.assertEqual(1, eos.nbfsls)
self.assertEqual(0, eos.nbstagerrm)
self.assertEqual(0, eos.nbattrset)
def test_run_only_once_one_fs_one_subdir_free_space(self):
mocksubdir = MockTreeNode("12345678")
......@@ -383,6 +398,7 @@ class GcTestCase(unittest.TestCase):
self.assertEqual(0, disk.nbgetfilesizeandctime)
self.assertEqual(0, eos.nbfsls)
self.assertEqual(0, eos.nbstagerrm)
self.assertEqual(0, eos.nbattrset)
gc = ctafstgcd.Gc(self.log, self.fqdn, disk, eos, self.config)
......@@ -393,6 +409,7 @@ class GcTestCase(unittest.TestCase):
self.assertEqual(0, disk.nbgetfilesizeandctime)
self.assertEqual(0, eos.nbfsls)
self.assertEqual(0, eos.nbstagerrm)
self.assertEqual(0, eos.nbattrset)
runonlyonce = True
gc.run(runonlyonce)
......@@ -404,6 +421,7 @@ class GcTestCase(unittest.TestCase):
self.assertEqual(0, disk.nbgetfilesizeandctime)
self.assertEqual(1, eos.nbfsls)
self.assertEqual(0, eos.nbstagerrm)
self.assertEqual(0, eos.nbattrset)
def test_run_only_once_one_fs_one_subdir_one_file_free_space(self):
mockfile = MockTreeNode("90abcdef")
......@@ -426,6 +444,7 @@ class GcTestCase(unittest.TestCase):
self.assertEqual(0, disk.nbgetfilesizeandctime)
self.assertEqual(0, eos.nbfsls)
self.assertEqual(0, eos.nbstagerrm)
self.assertEqual(0, eos.nbattrset)
gc = ctafstgcd.Gc(self.log, self.fqdn, disk, eos, self.config)
......@@ -436,6 +455,7 @@ class GcTestCase(unittest.TestCase):
self.assertEqual(0, disk.nbgetfilesizeandctime)
self.assertEqual(0, eos.nbfsls)
self.assertEqual(0, eos.nbstagerrm)
self.assertEqual(0, eos.nbattrset)
runonlyonce = True
gc.run(runonlyonce)
......@@ -447,6 +467,7 @@ class GcTestCase(unittest.TestCase):
self.assertEqual(0, disk.nbgetfilesizeandctime)
self.assertEqual(1, eos.nbfsls)
self.assertEqual(0, eos.nbstagerrm)
self.assertEqual(0, eos.nbattrset)
def test_run_only_once_one_fs_one_subdir_one_file_no_free_space_young_file(self):
mockfile = MockTreeNode("90abcdef")
......@@ -469,6 +490,7 @@ class GcTestCase(unittest.TestCase):
self.assertEqual(0, disk.nbgetfilesizeandctime)
self.assertEqual(0, eos.nbfsls)
self.assertEqual(0, eos.nbstagerrm)
self.assertEqual(0, eos.nbattrset)
self.config.minfreebytes = self.freebytes + 1
......@@ -481,6 +503,7 @@ class GcTestCase(unittest.TestCase):
self.assertEqual(0, disk.nbgetfilesizeandctime)
self.assertEqual(0, eos.nbfsls)
self.assertEqual(0, eos.nbstagerrm)
self.assertEqual(0, eos.nbattrset)
runonlyonce = True
gc.run(runonlyonce)
......@@ -492,6 +515,7 @@ class GcTestCase(unittest.TestCase):
self.assertEqual(1, disk.nbgetfilesizeandctime)
self.assertEqual(1, eos.nbfsls)
self.assertEqual(0, eos.nbstagerrm)
self.assertEqual(0, eos.nbattrset)
def test_run_only_once_one_fs_one_subdir_one_file_no_free_space_old_file(self):
mockfile = MockTreeNode("90abcdef")
......@@ -517,6 +541,7 @@ class GcTestCase(unittest.TestCase):
self.assertEqual(0, disk.nbgetfilesizeandctime)
self.assertEqual(0, eos.nbfsls)
self.assertEqual(0, eos.nbstagerrm)
self.assertEqual(0, eos.nbattrset)
self.config.minfreebytes = self.freebytes + 1
......@@ -529,6 +554,7 @@ class GcTestCase(unittest.TestCase):
self.assertEqual(0, disk.nbgetfilesizeandctime)
self.assertEqual(0, eos.nbfsls)
self.assertEqual(0, eos.nbstagerrm)
self.assertEqual(0, eos.nbattrset)
runonlyonce = True
gc.run(runonlyonce)
......@@ -540,6 +566,7 @@ class GcTestCase(unittest.TestCase):
self.assertEqual(1, disk.nbgetfilesizeandctime)
self.assertEqual(1, eos.nbfsls)
self.assertEqual(1, eos.nbstagerrm)
self.assertEqual(1, eos.nbattrset)
def test_run_only_once_one_fs_one_subdir_one_file_free_space_old_file(self):
mockfile = MockTreeNode("90abcdef")
......@@ -565,6 +592,7 @@ class GcTestCase(unittest.TestCase):
self.assertEqual(0, disk.nbgetfilesizeandctime)
self.assertEqual(0, eos.nbfsls)
self.assertEqual(0, eos.nbstagerrm)
self.assertEqual(0, eos.nbattrset)
self.config.minfreebytes = self.freebytes
......@@ -577,6 +605,7 @@ class GcTestCase(unittest.TestCase):
self.assertEqual(0, disk.nbgetfilesizeandctime)
self.assertEqual(0, eos.nbfsls)
self.assertEqual(0, eos.nbstagerrm)
self.assertEqual(0, eos.nbattrset)
runonlyonce = True
gc.run(runonlyonce)
......@@ -588,6 +617,7 @@ class GcTestCase(unittest.TestCase):
self.assertEqual(0, disk.nbgetfilesizeandctime)
self.assertEqual(1, eos.nbfsls)
self.assertEqual(0, eos.nbstagerrm)
self.assertEqual(0, eos.nbattrset)
if __name__ == '__main__':
suites = []
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

imprint | privacy | Conditions of Use