ctafstgcd.py 20 KB
Newer Older
Steven Murray's avatar
Steven Murray committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
#!/bin/python

# The CERN Tape Archive (CTA) project
# Copyright (C) 2015  CERN
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import argparse
import ConfigParser
import datetime
import distutils
23
import errno
Steven Murray's avatar
Steven Murray committed
24
25
26
27
28
29
30
31
32
33
34
35
36
import getpass
import logging
import logging.config
import os
import re
import socket
import subprocess
import sys
import time

class UserError(Exception):
  pass

37
38
39
class StagerrmError(Exception):
  pass

40
41
42
class AttrsetError(Exception):
  pass

Steven Murray's avatar
Steven Murray committed
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
class NoMgmHost(UserError):
  pass

class FileSizeAndCtime:
  def __init__(self):
    self.sizebytes = 0
    self.ctime = 0

class RealDisk:
  '''Class to faciliate unit-testing by wrapping the disk storage system.'''

  def __init__(self, log):
    self.log = log

  def listdir(self, path):
    return os.listdir(path)

  def isdir(self, path):
    return os.path.isdir(path)

  def isfile(self, path):
    return os.path.isfile(path)

66
  def get_file_size_and_ctime(self, path):
Steven Murray's avatar
Steven Murray committed
67
68
69
    try:
      statinfo = os.stat(path)

70
71
72
      file_size_and_ctime = FileSizeAndCtime()
      file_size_and_ctime.sizebytes = statinfo.st_size
      file_size_and_ctime.ctime = statinfo.st_ctime
Steven Murray's avatar
Steven Murray committed
73

74
      return file_size_and_ctime
Steven Murray's avatar
Steven Murray committed
75
    except Exception as err:
76
77
78
79
      # The file may have been deleted by the FST just before the stat
      if err.errno == errno.ENOENT:
        return None
      else:
80
        raise Exception('Failed to stat file: path={}: {}'.format(path, err))
Steven Murray's avatar
Steven Murray committed
81

82
  def get_free_bytes(self, path):
Steven Murray's avatar
Steven Murray committed
83
84
85
86
87
    statvfs = None
    try:
      statvfs = os.statvfs(path)
      return statvfs.f_frsize * statvfs.f_bavail
    except Exception as err:
88
      self.log.error('get_free_bytes: Failed to stat VFS for free space: path={}'.format(path))
Steven Murray's avatar
Steven Murray committed
89
90
91
92

class RealEos:
  '''Class to faciliate unit-testing by wrapping the EOS storage system.'''

93
  def __init__(self, log, mgm_host, xrdsecssskt):
Steven Murray's avatar
Steven Murray committed
94
    self.log = log
95
    self.mgm_host = mgm_host
Steven Murray's avatar
Steven Murray committed
96
97
98
    self.xrdsecssskt = xrdsecssskt

  def fsls(self):
99
100
    mgmurl = 'root://{}'.format(self.mgm_host)
    cmd = 'eos -r 0 0 {} fs ls -m'.format(mgmurl)
Steven Murray's avatar
Steven Murray committed
101
    env = os.environ.copy()
102
103
    env['XrdSecPROTOCOL'] = 'sss'
    env['XrdSecSSSKT'] = self.xrdsecssskt
Steven Murray's avatar
Steven Murray committed
104
105
106
107
    process = None
    try:
      process = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
    except Exception as err:
108
      raise Exception('Failed to execute \'{}\'": {}'.format(cmd, err))
Steven Murray's avatar
Steven Murray committed
109
110
111
112
113
114
115
    stdout,stderr = process.communicate()

    result = []

    if 0 != process.returncode:
      returncodestr = os.strerror(process.returncode)
      stderrstr = stderr.replace('\n', ' ').replace('\r', '').strip()
116
      self.log.error('fsls: Failed to execute {}: returncode={} returncodestr=\'{}\' stderr=\'{}\''
Steven Murray's avatar
Steven Murray committed
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
        .format(cmd, process.returncode, returncodestr, stderrstr))
    else:
      lines = stdout.splitlines();
      for l in lines:
        linedict = {}
        pairs = l.split()
        for p in pairs:
          splitpair = p.split('=')
          if 2 == len(splitpair):
            linedict[splitpair[0]] = splitpair[1]
        if linedict:
          result.append(linedict)

    return result

Steven Murray's avatar
Steven Murray committed
132
  def stagerrm(self, fxid):
133
134
    mgmurl = 'root://{}'.format(self.mgm_host)
    cmd = 'eos {} stagerrm fxid:{}'.format(mgmurl, fxid)
Steven Murray's avatar
Steven Murray committed
135
    env = os.environ.copy()
136
137
    env['XrdSecPROTOCOL'] = 'sss'
    env['XrdSecSSSKT'] = self.xrdsecssskt
Steven Murray's avatar
Steven Murray committed
138
139
140
141
    process = None
    try:
      process = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
    except Exception as err:
142
      raise Exception('Failed to execute \'{}\': {}'.format(cmd, err))
Steven Murray's avatar
Steven Murray committed
143
144
    stdout,stderr = process.communicate()

Steven Murray's avatar
Steven Murray committed
145
    if 0 != process.returncode:
146
      raise StagerrmError('\'{}\' returned non zero: returncode={}'.format(cmd, process.returncode))
Steven Murray's avatar
Steven Murray committed
147

148
  def attrset(self, name, value, fxid):
149
150
    mgmurl = 'root://{}'.format(self.mgm_host)
    args = ['eos', '-r', '0', '0', mgmurl, 'attr', 'set', '{}={}'.format(name, value), 'fxid:{}'.format(fxid)]
151
    env = os.environ.copy()
152
153
    env['XrdSecPROTOCOL'] = 'sss'
    env['XrdSecSSSKT'] = self.xrdsecssskt
154
155
156
157
    process = None
    try:
      process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
    except Exception as err:
158
      raise Exception('Failed to execute \'{}\': {}'.format(' '.join(args), err))
159
160
161
    stdout,stderr = process.communicate()

    if 0 != process.returncode:
162
      raise AttrsetError('\'{}\' returned non zero: returncode={}'.format(' '.join(args), process.returncode))
163

Steven Murray's avatar
Steven Murray committed
164
165
166
167
168
class SpaceTracker:
  '''Calculates the amount of effective free space in the file system of a given
  file or directory by querying the OS and taking into account the pending
  stagerrm requests to the EOS MGM.'''

169
  def __init__(self, disk, query_period_secs, path):
Steven Murray's avatar
Steven Murray committed
170
    self.disk = disk
171
    self.query_period_secs = query_period_secs
Steven Murray's avatar
Steven Murray committed
172
173
174
175
    self.path = path
    self.freebytes = None
    self.lastquerytimestamp = None

176
177
  def stagerrm_queued(self, file_size_bytes):
    self.freebytes = self.freebytes + file_size_bytes
Steven Murray's avatar
Steven Murray committed
178

179
  def get_free_bytes(self):
Steven Murray's avatar
Steven Murray committed
180
181
182
183
184
    now = time.time()

    if self.freebytes:
      elapsed = now - self.lastquerytimestamp

185
      if elapsed > self.query_period_secs:
Steven Murray's avatar
Steven Murray committed
186
        self.lastquerytimestamp = now
187
        self.freebytes = self.disk.get_free_bytes(self.path)
Steven Murray's avatar
Steven Murray committed
188
189
190

    else: # not self.freebytes:
      self.lastquerytimestamp = now
191
      self.freebytes = self.disk.get_free_bytes(self.path)
Steven Murray's avatar
Steven Murray committed
192
193
194
195
196
197
198

    return self.freebytes

class SpaceTrackers:
  '''Container and factory of SpaceTracker objects.  There is one SpaceTracker
  per filesystem mount-point being tracked.'''

199
  def __init__(self, log, disk, query_period_secs):
Steven Murray's avatar
Steven Murray committed
200
201
    self.log = log
    self.disk = disk
202
203
    self.query_period_secs = query_period_secs
    self.mnt_point_to_tracker = {}
Steven Murray's avatar
Steven Murray committed
204

205
206
  def get_tracker(self, path):
    mntpoint = self.get_mnt_point(path)
Steven Murray's avatar
Steven Murray committed
207

208
209
210
    if not mntpoint in self.mnt_point_to_tracker:
      self.mnt_point_to_tracker[mntpoint] = SpaceTracker(self.disk, self.query_period_secs, mntpoint)
      self.log.info('Tracking storage space of mount point: mntpoint={}'.format(mntpoint))
Steven Murray's avatar
Steven Murray committed
211

212
    return self.mnt_point_to_tracker[mntpoint]
Steven Murray's avatar
Steven Murray committed
213

214
215
  def get_nb_trackers(self):
    return len(self.mnt_point_to_tracker)
Steven Murray's avatar
Steven Murray committed
216

217
  def get_mnt_point(self, path):
Steven Murray's avatar
Steven Murray committed
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
    realpath = os.path.realpath(path)
    while not os.path.ismount(realpath):
      realpath = os.path.dirname(realpath)
    return realpath

class DummyLoggingHandler(logging.StreamHandler):
   '''Dummy logging handler that does nothing'''

   def __init__(self):
     logging.StreamHandler.__init__(self)

   def emit(self, record):
     pass

class GcConfig:
  '''The configuration of a cta-fst-gcd daemon.'''

  def __init__(self):
236
237
238
239
240
241
242
    self.log_file = ''
    self.mgm_host = ''
    self.eos_spaces = []
    self.eos_space_to_min_free_bytes = {}
    self.gc_age_secs = 0
    self.query_period_secs = 0
    self.main_loop_period_secs = 0
Steven Murray's avatar
Steven Murray committed
243
244
    self.xrdsecssskt = ''

245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
class PathAndEosSpace:
  def __init__(self):
    self.path = ''
    self.eos_space = ''

  def __init__(self, path, eos_space):
    self.path = path
    self.eos_space = eos_space

class MissingColonError(UserError):
  pass

class TooManyColonsError(UserError):
  pass

class MinFreeBytesNotSetError(UserError):
  pass

class MinFreeBytesError(UserError):
  pass

Steven Murray's avatar
Steven Murray committed
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
class Gc:
  '''Implements the cta-fst-gcd daemon that runs on an EOS FST and garbage
  collects EOS disk copies that have been safely stored to tape.

  The cta-fst-gcd daemon scans across every single EOS disk file on an FST.
  A file is garbage collected if:

  * The amount of free space on the corresponding file system is
    considered too low.

  * The file is considered old enough to be garbage collected.

  The cta-fst-gcd daemon garbage collects an EOS disk file by extracting the
  hexadecimal EOS file identifier from the local disk filename and then
  running eos stagerm fxid:<fid-hex>.'''

  def __init__(self, log, fqdn, disk, eos, config):
    self.log = log
    self.fqdn = fqdn
    self.disk = disk
    self.eos = eos
    self.config = config

289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
    self.local_file_system_paths = []
    self.space_trackers = SpaceTrackers(self.log, disk, self.config.query_period_secs)

    self.log.info('Config: log_file={}'.format(self.config.log_file))
    self.log.info('Config: mgm_host={}'.format(self.config.mgm_host))
    self.log.info('Config: eos_spaces={}'.format(' '.join(self.config.eos_spaces)))
    self.log.info('Config: eos_space_to_min_free_bytes={}'.format(config.eos_space_to_min_free_bytes_str))
    self.log.info('Config: gc_age_secs={}'.format(self.config.gc_age_secs))
    self.log.info('Config: absolute_max_age_secs={}'.format(self.config.absolute_max_age_secs))
    self.log.info('Config: query_period_secs={}'. format(self.config.query_period_secs))
    self.log.info('Config: main_loop_period_secs={}'. format(self.config.main_loop_period_secs))
    self.log.info('Config: xrdsecssskt={}'.format(self.config.xrdsecssskt))

  @staticmethod
  def parse_conf(conf_fp):
304
    try:
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
      parser = ConfigParser.ConfigParser()
      parser.readfp(conf_fp)
      config = GcConfig()
      config.log_file = parser.get('main', 'log_file')
      config.mgm_host = parser.get('main', 'mgm_host')
      config.eos_spaces = parser.get('main', 'eos_spaces').split()
      config.eos_space_to_min_free_bytes_str = parser.get('main', 'eos_space_to_min_free_bytes')
      config.eos_space_to_min_free_bytes = Gc.parse_eos_space_to_min_free_bytes(config.eos_space_to_min_free_bytes_str)
      config.gc_age_secs = parser.getint('main', 'gc_age_secs')
      config.absolute_max_age_secs = parser.getint('main', 'absolute_max_age_secs')
      config.query_period_secs = parser.getint('main', 'query_period_secs')
      config.main_loop_period_secs = parser.getint('main', 'main_loop_period_secs')
      config.xrdsecssskt = parser.get('main', 'xrdsecssskt')

      for eos_space in config.eos_spaces:
        # config.eos_space_to_min_free_bytes is a dictionary mapping EOS space
        # name to minimum number of free bytes
        if eos_space not in config.eos_space_to_min_free_bytes:
          raise MinFreeBytesNotSetError(
           'Error in eos_space_to_min_free_bytes value: ' \
           'Minimum number of free bytes has not been specified for EOS space {}: ' \
           'value=\'{}\''.format(eos_space, config.eos_space_to_min_free_bytes_str))

      return config
    except ConfigParser.Error as err:
      raise UserError(err)

  @staticmethod
  def parse_eos_space_to_min_free_bytes(str):
    # Dictionary mapping EOS space name to minimum number of free bytes
    eos_space_to_min_free_bytes = {}

    for maplet_str in str.split():
      maplet = maplet_str.split(':')
      if 1 >= len(maplet):
        raise MissingColonError('Syntax error in eos_space_to_min_free_bytes value' \
         ': Invalid maplet: Missing colon: maplet=\'{}\'"'.format(maplet_str))
      if 2 < len(maplet):
        raise TooManyColonsError('Syntax error in eos_space_to_min_free_bytes value: Invalid maplet' \
          ': Too many colons: maplet=\'{}\''.format(maplet_str))

      eos_space = maplet[0]
      min_free_bytes_str = maplet[1]

      min_free_bytes_int = 0
Steven Murray's avatar
Steven Murray committed
350
      try:
351
352
353
354
        min_free_bytes_int = int(min_free_bytes_str)
      except ValueError as err:
        raise MinFreeBytesError('Minimum number of free bytes is not a valid integer: eos_space={} value={}: {}'.
          format(eos_space, min_free_bytes_str, err))
Steven Murray's avatar
Steven Murray committed
355

356
357
358
      if 0 > min_free_bytes_int:
        raise MinFreeBytesError('Minimum number of free bytes is negative: eos_space={} value={}'.
          format(eos_space, min_free_bytes_int))
359

360
      eos_space_to_min_free_bytes[eos_space] = min_free_bytes_int
361

362
    return eos_space_to_min_free_bytes
363

364
365
366
367
  def process_all_fs(self):
    # Get the paths of the EOS file systems local to this FST and log them if
    # they have changed
    file_systems = None
368
    try:
369
      file_systems = self.eos.fsls()
370
    except Exception as err:
371
372
373
374
      self.log.error('process_all_fs: Failed to determine the EOS file systems local to this FST: {}'.format(err))

    if not file_systems:
      return
375

376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
    new_local_file_system_paths = [
      PathAndEosSpace(fs['path'], self.get_space_from_schedgroup(fs['schedgroup']))
      for fs in file_systems if
        'path' in fs and
        'host' in fs and self.fqdn == fs['host'] and
        'schedgroup' in fs and self.schedgroup_matches_eos_spaces(fs['schedgroup'])
    ]
    if new_local_file_system_paths != self.local_file_system_paths:
      self.local_file_system_paths = new_local_file_system_paths
      self.log_file_system_paths();

    for path_and_eos_space in self.local_file_system_paths:
      self.process_fs(path_and_eos_space.path, path_and_eos_space.eos_space)

  def log_file_system_paths(self):
    self.log.info('Number of local file systems is {}'.format(len(self.local_file_system_paths)))
    i = 0
    for path_and_eos_space in self.local_file_system_paths:
      self.log.info('Local file system {}: path={} eos_space={}'.format(i, path_and_eos_space.path, path_and_eos_space.eos_space))
      i = i + 1
Steven Murray's avatar
Steven Murray committed
396

397
  def process_fs(self, path, eos_space):
398
399
400
401
    fsfiles = []
    try:
      fsfiles = self.disk.listdir(path)
    except Exception as err:
402
      self.log.error('process_fs: Failed to list contents of filesystem: path={}: {}'.format(path, err))
403
404
      return

405
    sub_dirs = []
406
    try:
407
      sub_dirs = [os.path.join(path, f) for f in fsfiles
408
409
        if re.match('^[0-9A-Fa-f]{8}$', f) and self.disk.isdir(os.path.join(path, f))]
    except Exception as err:
410
      self.log.error('process_fs: Failed to determine sub directories of filesystem: path={}: {}'.format(path, err))
411
412
      return

413
414
    for sub_dir in sub_dirs:
      self.process_fs_sub_dir(sub_dir, eos_space)
Steven Murray's avatar
Steven Murray committed
415

416
417
418
419
420
  def schedgroup_matches_eos_spaces(self, schedgroup):
    for eos_space in self.config.eos_spaces:
      if re.match('^{}\.'.format(eos_space), schedgroup):
        return True
    return False
Steven Murray's avatar
Steven Murray committed
421

422
423
424
425
426
  def get_space_from_schedgroup(self, schedgroup):
    return re.match('^([^.]+)', schedgroup).group(0)

  def process_fs_sub_dir(self, sub_dir, eos_space):
    sub_dir_files = []
Steven Murray's avatar
Steven Murray committed
427
    try:
428
      sub_dir_files = self.disk.listdir(sub_dir)
Steven Murray's avatar
Steven Murray committed
429
    except Exception as err:
430
      self.log.error('process_fs_sub_dir: Failed to list contents of sub directory: sub_dir={}: {}'.format(sub_dir, err))
Steven Murray's avatar
Steven Murray committed
431

432
    fst_files = [f for f in sub_dir_files if re.match('^[0-9A-Fa-f]{8,}$', f) and self.disk.isfile(os.path.join(sub_dir, f))]
433
434
435
436
437
438
439
440
441
442
443
444
    for fst_file in fst_files:
      self.process_file(sub_dir, fst_file, eos_space)

  def process_file(self, sub_dir, fst_file, eos_space):
    fullpath = os.path.join(sub_dir, fst_file)
    file_size_and_ctime = None
    try:
      file_size_and_ctime = self.disk.get_file_size_and_ctime(fullpath)
    except Exception as err:
      self.log.error('process_file: Error calling self.disk.get_file_size_and_ctime(): {}'.format(err))

    if not file_size_and_ctime:
Steven Murray's avatar
Steven Murray committed
445
446
      return

447
448
    if eos_space not in self.config.eos_space_to_min_free_bytes:
      return
Steven Murray's avatar
Steven Murray committed
449

450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
    now = time.time()
    age_secs = now - file_size_and_ctime.ctime
    absolute_max_age_reached = age_secs > self.config.absolute_max_age_secs
    gc_age_reached = age_secs > self.config.gc_age_secs
    space_tracker = self.space_trackers.get_tracker(sub_dir)
    total_free_bytes = space_tracker.get_free_bytes()
    should_free_space = total_free_bytes < self.config.eos_space_to_min_free_bytes[eos_space]

    if absolute_max_age_reached or (should_free_space and gc_age_reached):
      try:
        bytes_required_before = 0
        if self.config.eos_space_to_min_free_bytes[eos_space] > total_free_bytes:
          bytes_required_before = self.config.eos_space_to_min_free_bytes[eos_space] - total_free_bytes
        self.eos.stagerrm(fst_file)
        space_tracker.stagerrm_queued(file_size_and_ctime.sizebytes)
        self.log.info('stagerrm: ' \
          'sub_dir={}, ' \
          'fxid={}, ' \
          'bytes_required_before={}, ' \
          'file_size_bytes={}, ' \
          'absolute_max_age_reached={}, ' \
          'should_free_space={}, ' \
          'gc_age_reached={}'
          .format(sub_dir, fst_file, bytes_required_before, file_size_and_ctime.sizebytes, absolute_max_age_reached,
            should_free_space, gc_age_reached))
        nowstr = datetime.datetime.now().strftime('%Y/%m/%d %H:%M:%S.%f')
        attrname = 'sys.retrieve.error'
        attrvalue = 'Garbage collected at {}'.format(nowstr)
        self.eos.attrset(attrname, attrvalue, fst_file)
      except StagerrmError as err:
        pass
      except Exception as err:
        self.log.error('process_file: {}'.format(err))
Steven Murray's avatar
Steven Murray committed
483

484
485
486
  def run(self, run_only_once = False):
    continue_main_loop = True
    while continue_main_loop:
Steven Murray's avatar
Steven Murray committed
487
      before = time.time()
488
      self.process_all_fs()
Steven Murray's avatar
Steven Murray committed
489
490
      after = time.time()
      period = after - before
491
492
      if period < self.config.main_loop_period_secs:
        sleeptime = self.config.main_loop_period_secs - period
Steven Murray's avatar
Steven Murray committed
493
494
        self.log.debug('Sleeping {} seconds'.format(sleeptime))
        time.sleep(sleeptime)
495
496
      if run_only_once:
        continue_main_loop = False
Steven Murray's avatar
Steven Murray committed
497
498
499
500
501
502
503
504
505

def main():
  programname = 'cta-fst-gcd'

  username = getpass.getuser()
  if 'daemon' != username:
    raise UserError('{} must be executed as user daemon and not user {}'.format(programname, username))

  parser = argparse.ArgumentParser()
506
  parser.add_argument('-c', '--config', default='/etc/cta/{}.conf'.format(programname), help='Configuration file path')
507
  parser.add_argument('-s', '--stdout', action='store_true', help='Print logs to standard output, overriding configuration file path')
Steven Murray's avatar
Steven Murray committed
508
509
  args = parser.parse_args()

510
511
512
513
514
515
516
517
518
519
520
521
  if not os.path.isfile(args.config):
    raise UserError('The configuration file {} is not a directory or does not exist'.format(args.config))
  if not os.access(args.config, os.R_OK):
    raise UserError('Cannot access for reading the configuration file {}'.format(args.config))

  conf_fp = open(args.config)

  try:
    config = Gc.parse_conf(conf_fp)
  except UserError as err:
    raise UserError('Error parsing configuration file {}: {}'.format(args.config, err))

Steven Murray's avatar
Steven Murray committed
522
523
524
  hostname = socket.gethostname()
  fqdn = socket.getfqdn()

525
526
527
528
529
530
  if args.stdout == True:
    # If --stdout has been given on the command line, use False as the log path to set output to stdout
    log = get_logger(hostname, programname, False)
  else:
    log = get_logger(hostname, programname, config.log_file)

Steven Murray's avatar
Steven Murray committed
531
532
533
  log.info('{} started'.format(programname))
  log.info('The fqdn of this machine is {}'.format(fqdn))

534
  eos = RealEos(log, config.mgm_host, config.xrdsecssskt)
Steven Murray's avatar
Steven Murray committed
535
536
537
538
  disk = RealDisk(log)
  gc = Gc(log, fqdn, disk, eos, config)
  gc.run()

539
def get_logger(hostname, programname, logpath):
Steven Murray's avatar
Steven Murray committed
540
541
  config = {}

542
543
544
545
  log_fmt = '%(asctime)s.%(msecs)03d000 ' + hostname + ' %(levelname)s ' + programname + \
    ':LVL="%(levelname)s" PID="%(process)d" TID="%(process)d" MSG="%(message)s"'
  log_date_fmt = '%Y/%m/%d %H:%M:%S'
  log_formatter = logging.Formatter(fmt = log_fmt, datefmt = log_date_fmt)
Steven Murray's avatar
Steven Murray committed
546

547
  log_handler = None
Steven Murray's avatar
Steven Murray committed
548

549
550
551
552
553
554
555
556
557
  if logpath == False:
    log_handler = logging.StreamHandler(stream = sys.stdout)
  else:
    logging_dir = os.path.dirname(logpath)
    if not os.path.isdir(logging_dir):
      raise UserError('The logging directory {} is not a directory or does not exist'.format(logging_dir))
    if not os.access(logging_dir, os.W_OK):
      raise UserError('The logging directory {} cannot be written to'.format(logging_dir))
    log_handler = logging.handlers.TimedRotatingFileHandler(filename = logpath, when = 'midnight')
Steven Murray's avatar
Steven Murray committed
558

559
560
561
  log_handler = logging.handlers.TimedRotatingFileHandler(filename = logpath, when = 'midnight')
  log_handler.setLevel(logging.INFO)
  log_handler.setFormatter(log_formatter)
Steven Murray's avatar
Steven Murray committed
562
563
564

  logger = logging.getLogger('gc')
  logger.setLevel(logging.INFO)
565
  logger.addHandler(log_handler)
Steven Murray's avatar
Steven Murray committed
566
567
568
569
570
571
572
573
574
575

  return logger

try:
  if __name__ == '__main__':
    main()
except UserError as err:
  print(err)
except KeyboardInterrupt:
  pass