Executing first underlying actions

diff --git a/deploy-new.py b/deploy-new.py
index f701bd530d29d0073fe1874937238fe423114943..32317a3523270b6062d236bef84c9f2d6f0f2a3d 100755 (executable)
--- a/deploy-new.py
+++ b/deploy-new.py
@@ -4,33 +4,29 @@
 # Standard modules
 import os
 import sys
-
+import logging
 
 # own modules:
 basedir = os.path.abspath(os.path.dirname(__file__))
 libdir = os.path.join(basedir, 'lib')
 
-#print("libdir: {!r}".format(libdir))
-
 sys.path.insert(0, libdir)
 
 
 import webhooks
 
-#from webhooks.errors import MigrationAppError
-
 from webhooks.deploy import WebhookDeployApp
 
 MY_APPNAME = 'deploy'
+LOG = logging.getLogger(MY_APPNAME)
 
 app = WebhookDeployApp(appname=MY_APPNAME)
 
-if app.verbose > 2 and 'REQUEST_METHOD' not in os.environ:
-    print("%s object:\n%s" % (app.__class__.__name__, app))
+if app.verbose > 2:
+    LOG.debug("{c} object:\n{o}".format(c=app.__class__.__name__, o=app))
 
 app()
 
 sys.exit(0)
 
-
 # vim: ts=4 et

diff --git a/deploy.yaml b/deploy.yaml
index b84e10a1f8689505e95ca4afa09ee6895a75e100..2179b9c613ea8554a42c75b2651554ec42ddf5af 100644 (file)
--- a/deploy.yaml
+++ b/deploy.yaml
@@ -1,7 +1,9 @@
 ---
 verbose: 0
+do_sudo: true
 log_dir: '/var/log/webhooks'
 default_email: 'frank@brehm-online.com'
+default_parent_dir: '/www/data'
 ignore_projects:
 mail_cc_addresses:
   - 'webmaster@pixelpark.com'
@@ -15,6 +17,12 @@ projects:
   puppet_modules:
     namespace: 'puppet'
     parent_dir: '/www/data/puppet-environment'
+  postfix_config:
+    namespace: 'ppadmin'
+    name: 'postfix_config'
+    parent_dir: '/www/data'
+    workdir: 'postfix_config'
+    branch: 'master'
 
 mail_bodies:
   special_chars: "Received special characters in module name"

diff --git a/lib/webhooks/common.py b/lib/webhooks/common.py
index 41abed7a990d7ecb3a7bb2f6c46b38e7fbb11459..75b2b04cfb2e6faa47a853f77734966c9495440f 100644 (file)
--- a/lib/webhooks/common.py
+++ b/lib/webhooks/common.py
@@ -19,7 +19,10 @@ import platform
 
 __version__ = '0.2.0'
 
-log = logging.getLogger(__name__)
+LOG = logging.getLogger(__name__)
+RE_YES = re.compile(r'^\s*(?:y(?:es)?|true)\s*$', re.IGNORECASE)
+RE_NO = re.compile(r'^\s*(?:no?|false|off)\s*$', re.IGNORECASE)
+
 
 # =============================================================================
 def pp(value, indent=4, width=99, depth=None):
@@ -140,6 +143,47 @@ def to_str(obj, encoding='utf-8'):
 
 
 
+# =============================================================================
+def to_bool(value):
+    """
+    Converter from string to boolean values (e.g. from configurations)
+    """
+
+    if not value:
+        return False
+
+    if isinstance(value, bool):
+        return value
+
+    try:
+        v_int = int(value)
+    except ValueError:
+        pass
+    except TypeError:
+        pass
+    else:
+        if v_int == 0:
+            return False
+        else:
+            return True
+
+    v_str = ''
+    if isinstance(value, str):
+        v_str = value
+    elif isinstance(value, bytes):
+        v_str = value.decode('utf-8')
+    else:
+        v_str = str(value)
+
+    if RE_YES.search(v_str):
+        return True
+
+    if RE_NO.search(v_str):
+        return False
+
+    return bool(value)
+
+
 # =============================================================================
 
 if __name__ == "__main__":

diff --git a/lib/webhooks/deploy.py b/lib/webhooks/deploy.py
index cc0edc033ea016c270ac4ec8c841733ee150eb95..66a35811442c4a1ba05a08ed3c2ed4b74463aeb6 100644 (file)
--- a/lib/webhooks/deploy.py
+++ b/lib/webhooks/deploy.py
@@ -16,6 +16,9 @@ import textwrap
 import datetime
 import json
 import traceback
+import copy
+import pipes
+import subprocess
 
 # Third party modules
 import yaml
@@ -23,12 +26,13 @@ import yaml
 # Own modules
 import webhooks
 
-from webhooks.common import pp, to_bytes
+from webhooks.common import pp, to_bytes, to_str, to_bool
 
 __version__ = webhooks.__version__
 LOG = logging.getLogger(__name__)
 DEFAULT_EMAIL = 'frank.brehm@pixelpark.com'
 DEFAULT_SENDER = 'Puppetmaster <{}>'.format(DEFAULT_EMAIL)
+DEFAULT_PARENT_DIR = '/www/data'
 
 
 # =============================================================================
@@ -40,6 +44,8 @@ class WebhookDeployApp(object):
     cgi_bin_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
     base_dir = os.path.dirname(cgi_bin_dir)
 
+    special_chars_re = re.compile(r'[^a-z0-9_\-]', re.IGNORECASE)
+
     mail_bodies = {
         'special_chars': "Received special characters in module name",
         'no_branch_dir': "Branch folder does not exist",
@@ -81,9 +87,12 @@ class WebhookDeployApp(object):
         self.namespace = None
         self.name = None
         self.full_name = None
+        self.git_ssh_url = None
+        self.do_sudo = True
 
         self.ignore_projects = []
 
+        self.default_parent_dir = '/www/data'
         self.default_email = DEFAULT_EMAIL
         self.mail_to_addresses = []
         self.mail_cc_addresses = [
@@ -94,6 +103,18 @@ class WebhookDeployApp(object):
 
         self._log_directory = os.sep + os.path.join('var', 'log', 'webhooks')
 
+        self.projects = {
+            'hiera': {
+                'namespace': 'puppet',
+                'name': 'hiera',
+                'parent_dir': '/www/data/puppet-hiera'
+            },
+            'puppet_modules': {
+                'namespace': 'puppet',
+                'parent_dir': '/www/data/puppet-environment'
+            }
+        }
+
         self.read_config()
         self.init_logging()
 
@@ -210,12 +231,20 @@ class WebhookDeployApp(object):
         if 'verbose' in config:
             self.verbose = config['verbose']
 
+        if 'do_sudo' in config:
+            self.do_sudo = to_bool(config['do_sudo'])
+
         if 'log_dir' in config and config['log_dir']:
             self._log_directory = config['log_dir']
 
         if 'default_email' in config and config['default_email']:
             self.default_email = config['default_email']
 
+        if 'default_parent_dir' in config and config['default_parent_dir']:
+            pdir = config['default_parent_dir']
+            if os.path.isabs(pdir):
+                self.default_parent_dir = pdir
+
         if 'ignore_projects' in config:
             if config['ignore_projects'] is None:
                 self.ignore_projects = []
@@ -240,6 +269,43 @@ class WebhookDeployApp(object):
             for key in config['mail_bodies'].keys():
                 self.mail_bodies[key] = config['mail_bodies'][key]
 
+        if 'projects' in config and isinstance(config['projects'], dict):
+            self.config_projects(config['projects'])
+
+    # -------------------------------------------------------------------------
+    def config_projects(self, project_cfg):
+
+        for project_key in project_cfg.keys():
+            if project_key == 'hiera':
+                cfg = project_cfg['hiera']
+                if 'namespace' in cfg and cfg['namespace'].strip():
+                    self.projects['hiera']['namespace'] = cfg['namespace'].strip()
+                if 'name' in cfg and cfg['name'].strip():
+                    self.projects['hiera']['name'] = cfg['name'].strip()
+                if 'parent_dir' in cfg and cfg['parent_dir']:
+                    self.projects['hiera']['parent_dir'] = cfg['parent_dir']
+                continue
+            if project_key == 'puppet_modules':
+                cfg = project_cfg['puppet_modules']
+                if 'namespace' in cfg and cfg['namespace'].strip():
+                    self.projects['puppet_modules']['namespace'] = cfg['namespace'].strip()
+                if 'parent_dir' in cfg and cfg['parent_dir']:
+                    self.projects['puppet_modules']['parent_dir'] = cfg['parent_dir']
+                continue
+            if project_key not in self.projects:
+                self.projects[project_key] = {}
+            cfg = project_cfg[project_key]
+            if 'namespace' in cfg and cfg['namespace'].strip():
+                self.projects[project_key]['namespace'] = cfg['namespace'].strip()
+            if 'name' in cfg and cfg['name'].strip():
+                self.projects[project_key]['name'] = cfg['name'].strip()
+            if 'parent_dir' in cfg and cfg['parent_dir']:
+                self.projects[project_key]['parent_dir'] = cfg['parent_dir']
+            if 'workdir' in cfg and cfg['workdir']:
+                self.projects[project_key]['workdir'] = cfg['workdir']
+            if 'branch' in cfg and cfg['branch'].strip():
+                self.projects[project_key]['branch'] = cfg['branch'].strip()
+
     # -------------------------------------------------------------------------
     def init_logging(self):
         """
@@ -390,8 +456,148 @@ class WebhookDeployApp(object):
         self.name = self.json_data['project']['name']
         self.full_name = self.json_data['project']['path_with_namespace']
 
+        if self.full_name in self.ignore_projects or self.name in self.ignore_projects:
+            LOG.info("Ignoring project {!r}.".format(self.full_name))
+            return True
+
+        if self.special_chars_re.search(self.name):
+            LOG.error(("Project {!r}: " + self.mail_bodies['special_chars']).format(
+                self.full_name))
+            return True
+
+        committers = []
+        timeformat = '%Y-%m-%dT%H:%M:%S%z'
+        for commit in self.json_data['commits']:
+            ts = commit['timestamp'].split('+')
+            ts_str = ts[0]
+            if len(ts) > 1:
+                ts_str += '+' + ts[1].replace(':', '').ljust(4, '0')
+            else:
+                ts_str += '+0000'
+            timestamp = datetime.datetime.strptime(ts_str, timeformat)
+            email = commit['author']['email']
+            committers.append((timestamp, email))
+        committers.sort()
+        if self.verbose > 1:
+            LOG.debug("Got committers: {}".format(pp(committers)))
+
+        self.mail_to_addresses.append(committers[-1][1])
+
+        if 'git_ssh_url' in self.json_data['project']:
+            self.git_ssh_url = self.json_data['project']['git_ssh_url']
+        else:
+            self.git_ssh_url = 'git@git.pixelpark.com:{ns}/{n}.git'.format(
+                ns=self.namespace, n=self.name)
+
+        if self.full_name == 'puppet/hiera':
+            return self.deploy_hiera()
+
+        if self.namespace == 'puppet':
+            return self.deploy_puppet_modules()
+
+        for project_key in self.projects.keys():
+            if project_key == 'hiera' or project_key == 'puppet_modules':
+                continue
+            cfg = copy.copy(self.projects[project_key])
+            if 'namespace' not in cfg:
+                LOG.debug("No namespace defined in project {n!r}:\n{p]".format(
+                    n=project_key, p=pp(cfg)))
+                continue
+            ns = cfg['namespace']
+            pname = project_key
+            if 'name' in cfg:
+                pname = cfg['name']
+            else:
+                cfg['name'] = project_key
+            full_name = ns + '/' + pname
+
+            LOG.debug("Checking for {!r} ...".format(full_name))
+
+            if self.full_name == full_name:
+                return self.deploy(cfg)
+
+        LOG.error("Could not find a definition for project {!r}.".format(self.full_name))
+
+        return True
+
+    # -------------------------------------------------------------------------
+    def deploy_hiera(self):
+
+        LOG.info("Deploying Hiera working directory ...")
+
+        return True
+
+    # -------------------------------------------------------------------------
+    def deploy_puppet_modules(self):
+
+        LOG.info("Deploying puppet modules working directory ...")
+
         return True
 
+    # -------------------------------------------------------------------------
+    def deploy(self, cfg):
+
+        ns = cfg['namespace']
+        pname = cfg['name']
+        full_name = ns + '/' + pname
+        parent_dir = self.default_parent_dir
+        if 'parent_dir' in cfg and cfg['parent_dir']:
+            parent_dir = cfg['parent_dir']
+        workdir = pname
+        if 'workdir' in cfg and cfg['workdir']:
+            workdir = cfg['workdir']
+        full_path = os.path.join(parent_dir, workdir)
+        branch = None
+        if 'branch' in cfg and cfg['branch']:
+            branch = cfg['branch']
+
+        LOG.info("Deploying working directory {f!r} for project {p!r} ...".format(
+            f=full_path, p=full_name))
+
+        if not os.access(parent_dir, os.F_OK):
+            LOG.error("Parent directory {d!r} for project {p!r} does not exists.".format(
+                    d=parent_dir, p=full_name))
+            return True
+
+        if not os.path.isdir(parent_dir):
+            LOG.error((
+                "Path for parent directory {d!r} for project {p!r} "
+                "is not a directory.").format(d=parent_dir, p=full_name))
+            return True
+
+        return self.ensure_workingdir(parent_dir, workdir, branch)
+
+    # -------------------------------------------------------------------------
+    def ensure_workingdir(self, parent_dir, workdir, branch=None):
+
+        os.chdir(parent_dir)
+        cmd = []
+        if self.do_sudo:
+            cmd = ['sudo', '-n']
+        if os.access(workdir, os.F_OK):
+            os.chdir(workdir)
+            cmd += ['git', 'pull']
+        else:
+            cmd += ['git', 'clone', self.git_ssh_url, workdir]
+            if branch:
+                cmd += ['-b', branch]
+        if self.verbose > 2:
+            LOG.debug("Cmd: {}".format(pp(cmd)))
+        cmd_str = ' '.join(map(lambda x: pipes.quote(x), cmd))
+        LOG.debug("Executing: {}".format(cmd_str))
+
+        git = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        (stdoutdata, stderrdata) = git.communicate()
+        ret_val = git.wait()
+
+        LOG.debug("Return value: {}".format(ret_val))
+        if stdoutdata:
+            LOG.debug("Output:\n{}".format(to_str(stdoutdata)))
+        else:
+            LOG.debug("No output.")
+        if stderrdata:
+            LOG.warn("Error messages on '{c}': {e}".format(c=cmd_str, e=to_str(stderrdata)))
+
 # =============================================================================
 
 if __name__ == "__main__":