From 958dfcaefa9be74337447fec093a4f79e9f09135 Mon Sep 17 00:00:00 2001 From: Frank Brehm Date: Mon, 23 Jan 2017 14:38:48 +0100 Subject: [PATCH] Adding current states of the webhook scripts. --- deploy.py | 219 +++++++++++++++++++++++++++++++++++++++++++++++++++ r10k_hook.py | 198 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 417 insertions(+) create mode 100755 deploy.py create mode 100755 r10k_hook.py diff --git a/deploy.py b/deploy.py new file mode 100755 index 0000000..4e60321 --- /dev/null +++ b/deploy.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# This script should be used as a web hook for git +# +# It receives push events as JSON-Data and synchronizes +# the local repository +# +# Author: Ivan Prikhodko +# Last change: 2014-10-01 15:40 + +import sys +import time +import json +import re +import os +import subprocess +import smtplib + + +import traceback + +DEBUG = True +LOGFILE = '/tmp/gitdeploy.txt' + +# only for debug: +# PREFIX = '/www/sites/puppetmaster.pixelpark.com/gitdeploy/cgi-bin' +# PREFIX_HIERA = '/www/sites/puppetmaster.pixelpark.com/gitdeploy/cgi-bin/puppet-hiera' + + +# correct prefix +PREFIX = '/www/data/puppet-environment' +PREFIX_HIERA = '/www/data/puppet-hiera' + +ESPECIALCHARS = "Received special characters in module name" +ENOBRANCHDIR = "Branch folder does not exist" +ENOMODSDIR = "Modules folder does not exist" + +EGITACCESSDENIED = "Access to remote repository was denied" + +#ignore_projects = ['nova', 'cinder', 'heat', 'neutron', 'glance', +# 'ceilometer', 'horizon'] + +ignore_projects = [] + +DEFAULT_EMAIL = 'ivan.prikhodko@pixelpark.com' + +# Correct: +mailTo = [] +mailCC = ['webmaster@pixelpark.com', DEFAULT_EMAIL] +#mailBCC = [DEFAULT_EMAIL] + +def sendEmail(text, moduleName='undefined'): + """ + Send text as EMail + """ + global mailTo + sender = DEFAULT_EMAIL + if not mailTo: + mailTo = [DEFAULT_EMAIL] + receivers = mailTo + mailCC + #receivers = ['ivan.prikhodko@pixelpark.com','philipp.dallig@pixelpark.com'] + + message = """From: Puppetmaster +To: %s +CC: %s +Subject: puppetmaster gitdeploy error + +Error while processing %s module: +%s. +""" % (", ".join(mailTo), ", ".join(mailCC), moduleName, text) + + # only for debug: + #fl = open('/tmp/gitdeploy.txt', 'a') + #fl.write("sender: %s receivers: %s, message: %s\n\n" % (sender, receivers, message)) + #fl.close() + + smtpObj = smtplib.SMTP('smtp.pixelpark.com', 25) + smtpObj.starttls() + smtpObj.sendmail(sender, receivers, message) + + +def index(): + """ + Parses JSON-Data and updates local git repository + """ + + + timeformat = '%Y-%m-%dT%H:%M:%S' + data = sys.stdin.read(); + myJsonData = json.loads(data) + ref = myJsonData[u'ref'].split('/')[-1] + name = myJsonData[u'repository'][u'name'] + if name in ignore_projects: + return "ignoring project %s from icehouse-branch" % name + + + # get committer's e-mail + committers = [] + for commit in myJsonData['commits']: + timestampString = commit['timestamp'].split('+')[0] + timestamp = time.strptime(timestampString, timeformat) + emailAddress = commit['author']['email'] + committers.append((timestamp, emailAddress)) + + committers.sort() + if committers: + mailTo.append(committers[-1][1]) + + + #url = myJsonData[u'repository'][u'url'] + if name == 'hiera': + url = 'git@git.pixelpark.com:puppet/hiera.git' + else: + url = 'git@git.pixelpark.com:puppet/%s.git' % name + branch = 'undefined' + + if DEBUG: + fl = open(LOGFILE, 'a') + fl.write("ref=%s name=%s url=%s PWD=%s\n" % (ref, name, url, os.getcwd())) + fl.close() + + if re.match("^dev.*", ref): + branch = 'development' + elif ref == 'master': + # sendEmail('test message 1, please ignore', name) + branch = 'test' + + # check if json parameters contain special characters + if re.search(r"[^a-zA-Z0-9_\-]", name): + sendEmail(ESPECIALCHARS, name) + return ESPECIALCHARS + + + if name != 'hiera': + os.chdir(PREFIX) + # test branch folder + if not os.access(branch, os.F_OK): + sendEmail(ENOBRANCHDIR, name) + return ENOBRANCHDIR + + # test folder 'modules' + modulesSubfolder = os.path.join(branch, 'modules') + if not os.access(modulesSubfolder, os.F_OK): + sendEmail(ENOMODSDIR, name) + return ENOMODSDIR + + os.chdir(modulesSubfolder) + if not os.access(name, os.F_OK): + # os.system('sudo git clone %s' % url) + #mdOutput = os.popen('sudo git clone %s 2>&1' % url) + if branch == 'test': + gitProcess = subprocess.Popen(['sudo', 'git', 'clone', '-b', 'master', url], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if DEBUG: + fl = open(LOGFILE, 'a') + fl.write("\n%s\n" % "cloning master, not default branch") + fl.close() + + + else: + gitProcess = subprocess.Popen(['sudo', 'git', 'clone', url], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = gitProcess.communicate() + if err: + if DEBUG: + fl = open(LOGFILE, 'a') + fl.write("\n%s\n" % err) + fl.close() + + + if 'Access denied' in err: + sendEmail(EGITACCESSDENIED, name) + return EGITACCESSDENIED + else: + os.chdir(name) + #os.system('sudo git pull') + gitProcess = subprocess.Popen(['sudo', 'git', 'pull'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = gitProcess.communicate() + if err: + if DEBUG: + fl = open(LOGFILE, 'a') + fl.write("\n%s\n" % err) + fl.close() + + if 'Access denied' in err: + sendEmail(EGITACCESSDENIED, name) + return EGITACCESSDENIED + + else: + # project hiera is a special case + if not os.access(PREFIX_HIERA, os.F_OK): + sendEmail(ENOMODSDIR, name) + return ENOMODSDIR + os.chdir(PREFIX_HIERA) + if not os.access(name, os.F_OK): + os.system('sudo -n git clone %s' % url) + else: + os.chdir(name) + os.system('sudo -n git pull') + + return data + +print "Content-Type: text/plain;charset=utf-8" +print + +print "Python CGI läuft" +try: + res = index() +except: + res = traceback.format_exc() + +if DEBUG: + fl = open(LOGFILE, 'a') + fl.write("\n" + time.asctime() + "\n") + fl.write(res + "\n\n") + fl.close() + diff --git a/r10k_hook.py b/r10k_hook.py new file mode 100755 index 0000000..7647833 --- /dev/null +++ b/r10k_hook.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# This script should be used as a web hook for git +# +# It receives push events as JSON-Data and synchronizes +# the local repository +# +# Author: Ivan Prikhodko +# Last change: 2014-10-01 15:40 + +import sys +import time +import json +import re +import os +import subprocess +import smtplib +sys.stderr = open('/tmp/r10k_gitdeploy.txt', 'a') + +import traceback + +DEBUG = True +LOGFILE = '/tmp/r10k_gitdeploy.txt' + +# only for debug: +# PREFIX = '/www/sites/puppetmaster.pixelpark.com/gitdeploy/cgi-bin' +# PREFIX_HIERA = '/www/sites/puppetmaster.pixelpark.com/gitdeploy/cgi-bin/puppet-hiera' + + +# correct prefix +PREFIX = '/www/data/puppet-environment' +PREFIX_HIERA = '/www/data/puppet-hiera' + +ESPECIALCHARS = "Received special characters in module name" +ENOBRANCHDIR = "Branch folder does not exist" +ENOMODSDIR = "Modules folder does not exist" + +EGITACCESSDENIED = "Access to remote repository was denied" + +PATH_TO_RESTART='/usr/share/puppet/rack/puppetmasterd/tmp/restart.txt' +#ignore_projects = ['nova', 'cinder', 'heat', 'neutron', 'glance', +# 'ceilometer', 'horizon'] + +ignore_projects = [] + +DEFAULT_EMAIL = 'ivan.prikhodko@pixelpark.com' + +# Correct: +mailTo = [] +mailCC = [DEFAULT_EMAIL] +#mailBCC = [DEFAULT_EMAIL] + +def sendEmail(text, moduleName='undefined'): + """ + Send text as EMail + """ + global mailTo + sender = DEFAULT_EMAIL + if not mailTo: + mailTo = [DEFAULT_EMAIL] + # TODO: uncomment + receivers = mailTo + mailCC + #receivers = ['ivan.prikhodko@pixelpark.com','philipp.dallig@pixelpark.com'] + + message = """From: Puppetmaster +To: %s +CC: %s +Subject: puppetmaster r10k webhook error + +Error while processing %s branch: +%s. +""" % (", ".join(mailTo), ", ".join(mailCC), moduleName, text) + + # only for debug: + #fl = open('/tmp/gitdeploy.txt', 'a') + #fl.write("sender: %s receivers: %s, message: %s\n\n" % (sender, receivers, message)) + #fl.close() + + smtpObj = smtplib.SMTP('smtp.pixelpark.com', 25) + smtpObj.starttls() + smtpObj.sendmail(sender, receivers, message) + + +def index(): + """ + Parses JSON-Data and updates local git repository + """ + + + timeformat = '%Y-%m-%dT%H:%M:%S' + data = sys.stdin.read(); + myJsonData = json.loads(data) + ref = myJsonData[u'ref'].split('/')[-1] + name = myJsonData[u'repository'][u'name'] + + # get committer's e-mail + committers = [] + for commit in myJsonData['commits']: + timestampString = commit['timestamp'].split('+')[0] + timestamp = time.strptime(timestampString, timeformat) + emailAddress = commit['author']['email'] + committers.append((timestamp, emailAddress)) + + committers.sort() + if committers: + mailTo.append(committers[-1][1]) + + + + r10kProcess = subprocess.Popen(['sudo', '-n', '/usr/local/bin/r10k', 'deploy', 'environment', ref, '--puppetfile', '-v'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + try: + out, err = r10kProcess.communicate() + + fl = open(LOGFILE, 'a') + fl.write("\nreturn code r10k:" + str(r10kProcess.returncode) + "\n") + fl.close() + + + if err and r10kProcess.returncode != 0: + error_message = "Error while executing r10k:\n\n" + err + sendEmail(error_message, ref) + else: + cacheClearProcess = subprocess.Popen(['curl', '-i', '--cert', '/var/lib/puppet/ssl/certs/puppetmaster01.pixelpark.com.pem', '--key', '/var/lib/puppet/ssl/private_keys/puppetmaster01.pixelpark.com.pem', '--cacert', '/var/lib/puppet/ssl/certs/ca.pem', '-X', 'DELETE', 'https://puppetmaster01.pixelpark.com:8140/puppet-admin-api/v1/environment-cache?environment='+ ref], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + try: + outCache, errCache = cacheClearProcess.communicate() + fl = open(LOGFILE, 'a') + fl.write("\nreturn code cacheClear:" + str(cacheClearProcess.returncode) + "\n") + fl.close() + if errCache and cacheClearProcess.returncode != 0: + error_messageCache = "Error while executing CacheClear:\n\n" + errCache + sendEmail(error_messageCache, ref) + except: + fl = open(LOGFILE, 'a') + fl.write("\n" + outCache + "\n") + fl.write("\n" + errCache + "\n") + fl.close() + +# os.system('sudo -n /bin/touch /usr/share/puppet/rack/puppetmasterd/tmp/restart.txt') + except: + + fl = open(LOGFILE, 'a') + fl.write("\n" + out + "\n") + fl.write("\n" + err + "\n") + fl.close() + + + fl = open(LOGFILE, 'a') + fl.write("\nout:" + out + "\n") + fl.write("\nerror:" + err + "\n") + fl.close() + + + + + # elif ref == 'master': + # # sendEmail('test message 1, please ignore', name) + # branch = 'test' + + # check if json parameters contain special characters + # if re.search(r"[^a-zA-Z0-9_\-]", name): + # sendEmail(ESPECIALCHARS, name) + # return ESPECIALCHARS + + + # stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + return data + +print "Content-Type: text/plain;charset=utf-8" +print + + +# try: +# res = index() +# except: +# res = traceback.format_exc() + + +try: + os.environ['LANG'] = 'de_DE.utf8' + os.environ['LC_CTYPE'] = 'de_DE.utf8' + fifo = open("/tmp/ivanslog", 'w') + fifo.write(repr(os.environ)+'\n') + fifo.close() + + res = index() + + fl = open(LOGFILE, 'a') + fl.write("\n" + time.asctime() + "\n") + fl.write(res + "\n\n") + fl.close() +except: + print traceback.format_exc() + +print "Python CGI läuft" -- 2.39.5