From: Frank Brehm Date: Fri, 24 Mar 2017 10:49:04 +0000 (+0100) Subject: Implemented rotation of status files X-Git-Tag: 0.1.2~218 X-Git-Url: https://git.uhu-banane.net/?a=commitdiff_plain;h=4d4be4ff1147728fb97af9554b450320b9e63431;p=pixelpark%2Fadmin-tools.git Implemented rotation of status files --- diff --git a/pp_lib/quota_check.py b/pp_lib/quota_check.py index f7a9e76..7c9d2b1 100644 --- a/pp_lib/quota_check.py +++ b/pp_lib/quota_check.py @@ -22,6 +22,9 @@ import copy import glob import stat import pipes +import gzip +import shutil +import time from subprocess import Popen, PIPE @@ -38,7 +41,7 @@ from .common import pp, terminal_can_colors, to_bytes, to_bool, to_str from .cfg_app import PpCfgAppError, PpConfigApplication -__version__ = '0.4.3' +__version__ = '0.5.1' LOG = logging.getLogger(__name__) UTC = datetime.timezone.utc @@ -67,6 +70,8 @@ class PpQuotaCheckApp(PpConfigApplication): du_line_re = re.compile(r'^\s*(\d+)\s+(.*)') + default_max_age = 365.25 * 4 * 24 * 60 * 60 + # ------------------------------------------------------------------------- def __init__(self, appname=None, version=__version__): @@ -80,11 +85,13 @@ class PpQuotaCheckApp(PpConfigApplication): self.statusfile_base = self.default_statusfile_base self.statusfile = os.path.join(self.status_dir, self.statusfile_base) self.status_data = {} + self.max_age = self.default_max_age self.passwd_data = {} self.map_uid = {} self.now = datetime.datetime.now(UTC) self.du_cmd = self.get_command('du', quiet=True) + self.do_statistics = False description = textwrap.dedent('''\ This checks the utilization of the home directories on the NFS server @@ -122,6 +129,15 @@ class PpQuotaCheckApp(PpConfigApplication): help="Quota value in MB (default: {} MB).".format(def_mb), ) + self.arg_parser.add_argument( + '-S', '--stats', + action="store_true", dest="stats", + help=( + "Generate statistics, mail them to the administrators and " + "rotate the status data file. Without this option the current " + "utilization is determined and saved in the status data file."), + ) + # ------------------------------------------------------------------------- def perform_config(self): @@ -209,6 +225,8 @@ class PpQuotaCheckApp(PpConfigApplication): if cmdline_quota is not None: self.quota_kb = cmdline_quota * 1024 + self.do_statistics = bool(getattr(self.args, 'stats', False)) + # ------------------------------------------------------------------------- def _run(self): @@ -219,6 +237,10 @@ class PpQuotaCheckApp(PpConfigApplication): self.write_status_data() + if self.do_statistics: + self.perform_statistics() + self.compress_old_status_files() + # ------------------------------------------------------------------------- def pre_run(self): """ @@ -276,18 +298,6 @@ class PpQuotaCheckApp(PpConfigApplication): LOG.debug("Status from {f!r}:\n{s}".format( f=self.statusfile, s=pp(status))) - if 'checks' in status and 'data' in status['checks']: - dates = [] - for date in status['checks']['data'].keys(): - dates.append(date) - if dates: - dates.sort() - first_date = dates[0].replace(tzinfo=UTC) - tdiff = self.now - first_date -# if tdiff.days > 7 or len(dates) > 5: -# self.rotate_status_file(dates[-1]) -# return {} - return status # ------------------------------------------------------------------------- @@ -301,11 +311,60 @@ class PpQuotaCheckApp(PpConfigApplication): file_stat = os.stat(self.statusfile) date = datetime.datetime.utcfromtimestamp(file_stat.st_mtime) (stem, ext) = os.path.splitext(self.statusfile) + new_fname = "{s}.{d}{e}".format( s=stem, d=date.strftime('%Y-%m-%d_%H:%M:%S'), e=ext) LOG.info("Renaming {o!r} -> {n!r}.".format(o=self.statusfile, n=new_fname)) os.rename(self.statusfile, new_fname) + # ------------------------------------------------------------------------- + def compress_old_status_files(self): + + (stem, ext) = os.path.splitext(self.statusfile_base) + search_base = "{s}.20*{e}".format(s=stem, e=ext) + seach_pattern = os.path.join(self.status_dir, search_base) + files = glob.glob(seach_pattern) + if len(files) <= 1: + return + + files.sort() + for filename in files[:-1]: + file_stat = os.stat(filename) + if not file_stat.st_size: + LOG.debug("Not compressing {!r} because of zero size.".format(filename)) + continue + LOG.info("Compressing {!r} ...".format(filename)) + new_name = filename + '.gz' + with open(filename, 'rb') as f_in: + with gzip.open(new_name, 'wb') as f_out: + shutil.copyfileobj(f_in, f_out) + shutil.copystat(filename, new_name) + LOG.debug("Removing {!r} ...".format(filename)) + os.remove(filename) + + files_to_remove = [] + files = glob.glob(seach_pattern) + search_base = "{s}.20*{e}.gz".format(s=stem, e=ext) + seach_pattern = os.path.join(self.status_dir, search_base) + files += glob.glob(seach_pattern) + files.sort() + # Removing all files older 4 years + limit_age = time.time() - self.max_age + limit_age_dt = datetime.datetime.fromtimestamp(limit_age, UTC) + LOG.info("Removing all status files older than {!r} ...".format( + limit_age_dt.isoformat(' '))) + + for filename in files[:-1]: + if not os.path.isfile(filename): + continue + file_stat = os.stat(filename) + if file_stat.st_mtime < limit_age: + files_to_remove.append(filename) + + for filename in files_to_remove: + LOG.info("Removing {!r} ...".format(filename)) + os.remove(filename) + # ------------------------------------------------------------------------- def write_status_data(self): @@ -506,6 +565,17 @@ class PpQuotaCheckApp(PpConfigApplication): self.send_mail(subject, body) + # ------------------------------------------------------------------------- + def perform_statistics(self): + + + + + # Rotate status file and rewrite an empty status file + self.rotate_status_file(self.now) + self.status_data = {} + self.status_data['last_check'] = self.now + self.write_status_data() # =============================================================================