From f483b3e9fc4661c6bf4e68968e94ccb1888516f2 Mon Sep 17 00:00:00 2001 From: Frank Brehm Date: Wed, 27 Feb 2019 18:28:58 +0100 Subject: [PATCH] Bugfixes, Finishing lib/webhooks/get_forge_modules.py --- lib/webhooks/base_app.py | 5 +- lib/webhooks/forge/__init__.py | 24 ++- lib/webhooks/forge/cur_mod_release_info.py | 4 +- lib/webhooks/forge/mod_dict.py | 218 ++++++++++++++++++++- lib/webhooks/forge/mod_info.py | 28 +-- lib/webhooks/forge/mod_release_info.py | 2 +- lib/webhooks/get_forge_modules.py | 4 +- lib/webhooks/module_meta_info.py | 6 +- 8 files changed, 249 insertions(+), 42 deletions(-) diff --git a/lib/webhooks/base_app.py b/lib/webhooks/base_app.py index 620f039..c96b49f 100644 --- a/lib/webhooks/base_app.py +++ b/lib/webhooks/base_app.py @@ -517,8 +517,9 @@ class BaseHookApp(BaseApplication): def cachefile(self): """The filename of the cacheing file.""" if self._cachefile.is_absolute(): - return self._cachefile.resolve() - return self.data_dir.joinpath(self._cachefile).resolve() + return pathlib.Path(os.path.abspath(str(self._cachefile))) + f = self.data_dir.joinpath(self._cachefile) + return pathlib.Path(os.path.abspath(str(f))) # ------------------------------------------------------------------------- def as_dict(self, short=True): diff --git a/lib/webhooks/forge/__init__.py b/lib/webhooks/forge/__init__.py index 364c3ec..fa68986 100644 --- a/lib/webhooks/forge/__init__.py +++ b/lib/webhooks/forge/__init__.py @@ -27,7 +27,7 @@ from ..base_module_info import BaseModuleInfoError, BaseModuleInfo from ..module_meta_info import ModuleMetadata -__version__ = '0.1.0' +__version__ = '0.2.0' LOG = logging.getLogger(__name__) @@ -183,24 +183,28 @@ class BaseForgeObject(FbBaseObject): @classmethod def from_data(cls, data, appname=None, verbose=0, base_dir=None): - owner = cls(appname=appname, verbose=verbose, base_dir=base_dir) + if verbose > 3: + LOG.debug(_("Trying to get data for {} from:").format( + cls.__name__) + '\n' + pp(data)) + + obj = cls(appname=appname, verbose=verbose, base_dir=base_dir) - owner.apply_data(data) + obj.apply_data(data) - owner.initialized = True + obj.initialized = True if verbose > 3: - LOG.debug(_("Got {}:").format(cls.__name__) + '\n' + pp(owner.as_dict())) + LOG.debug(_("Got {}:").format(cls.__name__) + '\n' + pp(obj.as_dict())) - return owner + return obj # ------------------------------------------------------------------------- def apply_data(self, data): - if 'gravatar_id' in data: - self.gravatar_id = data['gravatar_id'] - if 'username' in data: - self.username = data['username'] + if 'slug' in data: + self.slug = data['slug'] + if 'uri' in data: + self.uri = data['uri'] # ============================================================================= diff --git a/lib/webhooks/forge/cur_mod_release_info.py b/lib/webhooks/forge/cur_mod_release_info.py index c8b8e73..c4bd10e 100644 --- a/lib/webhooks/forge/cur_mod_release_info.py +++ b/lib/webhooks/forge/cur_mod_release_info.py @@ -35,7 +35,7 @@ from .mod_release_info import ModuleReleaseInfo from .base_module_info import BaseForgeModuleInfo -__version__ = '0.1.0' +__version__ = '0.1.1' LOG = logging.getLogger(__name__) @@ -424,7 +424,7 @@ class CurrentModuleReleaseInfo(ModuleReleaseInfo): if 'reference' in data: self.reference = data['reference'] if 'updated_at' in data: - self.reference = data['updated_at'] + self.updated_at = data['updated_at'] if 'validation_score' in data: self.validation_score = data['validation_score'] diff --git a/lib/webhooks/forge/mod_dict.py b/lib/webhooks/forge/mod_dict.py index daec854..d4e899f 100644 --- a/lib/webhooks/forge/mod_dict.py +++ b/lib/webhooks/forge/mod_dict.py @@ -10,22 +10,34 @@ from __future__ import absolute_import # Standard modules import logging - +import os +import json +import errno +import sys +import pwd +import grp +import fcntl + +from pathlib import Path from collections import MutableMapping from functools import cmp_to_key # Third party modules +from six import reraise + # Own modules -from fb_tools.common import to_bool +from fb_tools.common import to_bool, to_bytes from fb_tools.obj import FbBaseObject from .mod_info import ForgeModuleInfo +from ..base_app import BaseHookError + from ..xlate import XLATOR -__version__ = '0.1.0' +__version__ = '0.2.0' LOG = logging.getLogger(__name__) @@ -33,6 +45,44 @@ _ = XLATOR.gettext ngettext = XLATOR.ngettext +# ============================================================================= +class ForgeModuleDictError(BaseHookError): + + pass + +# ============================================================================= +class ParentNotExistingError(ForgeModuleDictError, IOError): + + # ------------------------------------------------------------------------- + def __init__(self, dirname): + + msg = _("Parent directory of forge modules cache is not existing") + super(ParentNotExistingError, self).__init__( + errno.ENOENT, msg, str(dirname)) + + +# ============================================================================= +class ParentNotDirError(ForgeModuleDictError, IOError): + + # ------------------------------------------------------------------------- + def __init__(self, dirname): + + msg = _("Parent directory of forge modules cache is not a directory") + super(ParentNotDirError, self).__init__( + errno.ENOTDIR, msg, str(dirname)) + + +# ============================================================================= +class ParentAccessError(ForgeModuleDictError, IOError): + + # ------------------------------------------------------------------------- + def __init__(self, dirname): + + msg = _("Parent directory of forge modules cache is not writeable") + super(ParentAccessError, self).__init__( + errno.EACCES, msg, str(dirname)) + + # ============================================================================= class ForgeModuleDict(MutableMapping, FbBaseObject): """ @@ -51,14 +101,22 @@ class ForgeModuleDict(MutableMapping, FbBaseObject): msg_empty_key_error = _("Empty key {!r} is not allowed.") msg_no_modinfo_dict = _("Object {{!r}} is not a {} object.").format('ForgeModuleDict') + root_path = Path(os.sep) + default_data_dir = root_path / 'var' / 'lib' / 'webhooks' + default_data_file = Path('forge-modules.js') + default_json_indent = 4 # ------------------------------------------------------------------------- # __init__() method required to create instance from class. def __init__( - self, appname=None, verbose=0, version=__version__, base_dir=None, - sort_by_name=False, *args, **kwargs): + self, appname=None, verbose=0, version=__version__, base_dir=None, json_indent=None, + data_dir=None, data_file=None, simulate=None, sort_by_name=False, *args, **kwargs): + self._data_dir = self.default_data_dir + self._data_file = self.default_data_file + self._json_indent = self.default_json_indent self._map = dict() + self._simulate = False self._sort_by_name = False super(ForgeModuleDict, self).__init__( @@ -67,10 +125,82 @@ class ForgeModuleDict(MutableMapping, FbBaseObject): ) self.sort_by_name = sort_by_name + if data_dir: + self.data_dir = data_dir + if data_file: + self.data_file = data_file + if json_indent is not None: + self.json_indent = json_indent + + if simulate is not None: + self.simulate = simulate for arg in args: self.append(arg) + # ----------------------------------------------------------- + @property + def data_dir(self): + """The directory containing some volatile data.""" + return self._data_dir + + @data_dir.setter + def data_dir(self, value): + if value is None: + msg = _("The value of {!r} may not be None.").format('data_dir') + raise TypeError(msg) + if str(value).strip() == '': + msg = _("The value of {!r} may not be empty.").format('data_dir') + raise ValueError(msg) + d = Path(value) + if not d.is_absolute(): + msg = _("The value of {!r} must be an absolute pathname.").format('data_dir') + raise ValueError(msg) + self._data_dir = Path(value) + + # ----------------------------------------------------------- + @property + def data_file(self): + """The file containing the cached forge module info.""" + return self._data_file + + @data_file.setter + def data_file(self, value): + if value is None: + msg = _("The value of {!r} may not be None.").format('data_file') + raise TypeError(msg) + if str(value).strip() == '': + msg = _("The value of {!r} may not be empty.").format('data_file') + raise ValueError(msg) + self._data_file = Path(value) + + # ----------------------------------------------------------- + @property + def simulate(self): + """A flag describing, that writing the cache file should not be executed.""" + return self._simulate + + @simulate.setter + def simulate(self, value): + self._simulate = to_bool(value) + + # ----------------------------------------------------------- + @property + def json_indent(self): + """The indention of the generated JSON output file. May be a string or integer.""" + return self._json_indent + + @json_indent.setter + def json_indent(self, value): + if value is None: + msg = _("The value of {!r} may not be None.").format('json_indent') + raise TypeError(msg) + if not isinstance(value, (str, int)): + msg = _("Wrong datatype {t!r} as a value for {w!r}.").format( + t=value.__class__.__name__, w='json_indent') + raise TypeError(msg) + self._json_indent = value + # ----------------------------------------------------------- @property def sort_by_name(self): @@ -105,6 +235,10 @@ class ForgeModuleDict(MutableMapping, FbBaseObject): res = super(ForgeModuleDict, self).as_dict(short=short) + res['data_dir'] = self.data_dir + res['data_file'] = self.data_file + res['json_indent'] = self.json_indent + res['simulate'] = self.simulate res['sort_by_name'] = self.sort_by_name res['items'] = {} res['keys'] = [] @@ -335,6 +469,80 @@ class ForgeModuleDict(MutableMapping, FbBaseObject): res.append(self._map[full_name].as_dict(short)) return res + # ------------------------------------------------------------------------- + def to_data(self): + """Returning a dict, which can be used to re-instantiate this module info.""" + + res = {} + + for full_name in self.keys(): + res[full_name] = self._map[full_name].to_data() + return res + + # ------------------------------------------------------------------------- + def write_file(self, output_file=None): + + if output_file is None: + output_file = self.data_file + if not output_file.is_absolute(): + output_file = self.data_dir / self.data_file + #output_file = output_file.resolve() + output_file = Path(os.path.abspath(str(output_file))) + fd = None + + if not output_file.parent.exists(): + raise ParentNotExistingError(output_file.parent) + if not output_file.parent.is_dir(): + raise ParentNotDirError(output_file.parent) + if not os.access(str(output_file.parent), os.W_OK): + raise ParentAccessError(output_file.parent) + + tmp_file = Path(str(output_file) + '.new') + data = self.to_data() + dump = json.dumps(data, indent=self.json_indent) + + LOG.info(_("Trying to open {!r} exclusive ...").format(str(tmp_file))) + + if self.simulate: + LOG.info(_("Simulation mode, cache file will not be written.")) + return + + try: + fd = os.open(str(tmp_file), os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644) + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError as e: + msg = _("Error on creating file {f!r}: {e}").format(f=tmp_file, e=e) + if e.errno == errno.EEXIST: + LOG.warning(msg) + return None + else: + error_tuple = sys.exc_info() + reraise(ForgeModuleDictError, msg, error_tuple[2]) + + try: + os.write(fd, to_bytes(dump)) + finally: + os.close(fd) + + if output_file.exists() and not os.geteuid(): + fstat = output_file.stat() + user = fstat.st_uid + try: + user = pwd.getpwuid(fstat.st_uid).pw_name + except KeyError: + user = fstat.st_uid + group = fstat.st_gid + try: + group = grp.getgrgid(fstat.st_gid).gr_name + except KeyError: + group = fstat.st_gid + LOG.debug("Chowning {f!r} to {u}:{g} ...".format( + f=tmp_file, u=user, g=group)) + os.chown(str(tmp_file), fstat.st_uid, fstat.st_gid) + + LOG.info(_("Renaming {src!r} => {tgt!r}.").format( + src=str(tmp_file), tgt=str(output_file))) + tmp_file.rename(output_file) # ============================================================================= diff --git a/lib/webhooks/forge/mod_info.py b/lib/webhooks/forge/mod_info.py index 025ff5f..aaf4535 100644 --- a/lib/webhooks/forge/mod_info.py +++ b/lib/webhooks/forge/mod_info.py @@ -31,7 +31,7 @@ from ..base_module_info import BaseModuleInfoError, BaseModuleInfo from ..module_meta_info import ModuleMetadata -from . import parse_forge_date +from . import parse_forge_date, ForgeModuleInfoError from .mod_release_info import ModuleReleaseInfo from .mod_release_list import ModuleReleaseList @@ -39,7 +39,7 @@ from .cur_mod_release_info import CurrentModuleReleaseInfo from .owner_info import ForgeOwnerInfo -__version__ = '0.1.0' +__version__ = '0.2.0' LOG = logging.getLogger(__name__) @@ -319,19 +319,6 @@ class ForgeModuleInfo(BaseModuleInfo): return self._supported = to_bool(value) - # ------------------------------------------------------------------------- - @property - def supported(self): - """Is this forge module supported by Puppetlabs?.""" - return self._supported - - @supported.setter - def supported(self, value): - if value is None: - self._supported = None - return - self._supported = to_bool(value) - # ------------------------------------------------------------------------- @property def updated_at(self): @@ -423,9 +410,11 @@ class ForgeModuleInfo(BaseModuleInfo): for prop_name in ( 'created_at', 'deprecated_at', 'deprecated_for', 'downloads', 'endorsement', 'feedback_score', 'homepage_url', 'issues_url', 'module_group', 'slug', - 'superseded_by', 'supported', 'updated_at', 'uri'): + 'superseded_by', 'updated_at', 'uri'): if prop_name in data and data[prop_name]: setattr(self, prop_name, data[prop_name]) + if 'supported' in data: + self.supported = data['supported'] if 'current_release' in data and data['current_release']: self.current_release = CurrentModuleReleaseInfo.from_data( @@ -519,17 +508,20 @@ class ForgeModuleInfo(BaseModuleInfo): return None data = response.json() - if verbose > 4: + if verbose > 3: LOG.debug("Performing forge data:\n" + pp(data)) module_info.apply_data(data) + if verbose > 2: + LOG.debug(_("Got {}:").format(cls.__name__) + '\n' + pp(module_info.as_dict())) + if module_info.superseded_by: subst = module_info.superseded_by if verbose > 2: LOG.debug("Superseded info:\n" + pp(subst)) if 'slug' in subst: subst = subst['slug'] - LOG.warning(_( + LOG.info(_( "Module {c!r} is deprecated at Puppet forge and should be substituted " "by module {n!r}.").format(c=module_info.slug, n=subst)) diff --git a/lib/webhooks/forge/mod_release_info.py b/lib/webhooks/forge/mod_release_info.py index b8238fd..59bfeca 100644 --- a/lib/webhooks/forge/mod_release_info.py +++ b/lib/webhooks/forge/mod_release_info.py @@ -262,7 +262,7 @@ class ModuleReleaseInfo(BaseForgeObject): self.file_uri = data['file_uri'] if 'slug' in data and data['slug']: self.slug = data['slug'] - if 'supported' in data and data['supported']: + if 'supported' in data: self.supported = data['supported'] if 'uri' in data and data['uri']: self.uri = data['uri'] diff --git a/lib/webhooks/get_forge_modules.py b/lib/webhooks/get_forge_modules.py index e8181f6..5d2dccd 100644 --- a/lib/webhooks/get_forge_modules.py +++ b/lib/webhooks/get_forge_modules.py @@ -121,7 +121,8 @@ class GetForgeModulesApp(BaseHookApp): def _init_forge_module_dict(self): self.forge_modules = ForgeModuleDict( - appname=self.appname, verbose=self.verbose, base_dir=self.base_dir) + appname=self.appname, verbose=self.verbose, base_dir=self.base_dir, + data_dir=self.data_dir, simulate=self.simulate) # ------------------------------------------------------------------------- def perform_arg_parser(self): @@ -217,6 +218,7 @@ class GetForgeModulesApp(BaseHookApp): # LOG.debug("Found modules:\n{}".format(pp(self.modules.as_list()))) #self.write_cache_file() + self.forge_modules.write_file() print() d = datetime.datetime.now(LOCALTZ) diff --git a/lib/webhooks/module_meta_info.py b/lib/webhooks/module_meta_info.py index f8a59a7..2852477 100644 --- a/lib/webhooks/module_meta_info.py +++ b/lib/webhooks/module_meta_info.py @@ -27,7 +27,7 @@ from fb_tools.obj import FbBaseObjectError, FbBaseObject from .xlate import XLATOR -__version__ = '0.5.2' +__version__ = '0.5.3' LOG = logging.getLogger(__name__) @@ -414,7 +414,7 @@ class ModuleMetadata(FbBaseObject): return new # ------------------------------------------------------------------------- - def to_data_dict(self): + def to_data(self): data = {} data['name'] = self.name @@ -451,7 +451,7 @@ class ModuleMetadata(FbBaseObject): # ------------------------------------------------------------------------- def to_json(self, indent=None): - data = self.to_data_dict() + data = self.to_data() ret = json.dumps(data, indent=indent, sort_keys=True) if self.verbose > 4: LOG.debug("ModuleMetadata as JSON:\n{}".format(ret)) -- 2.39.5