--- /dev/null
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+import re
+import sys
+import git
+import atexit
+import shutil
+import subprocess
+import logging
+import platform
+import smtplib
+from glob import glob
+from logging import Formatter
+from ftplib import FTP
+from multiprocessing import cpu_count
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy import Table, Column, Integer, String, MetaData, Sequence
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm.exc import NoResultFound
+
+GIT = '/usr/bin/git'
+MAKE_KPKG = '/usr/bin/make-kpkg'
+DEFAULT_PARALLEL_JOBS = cpu_count() + 1
+
+BUILD_ARCH_MAP = {
+ 'x86_64': 'amd64',
+ 'i386': '686'
+}
+
+BUILD_ARCH = BUILD_ARCH_MAP.get(platform.machine(), '686')
+
+CWD = os.environ.get('WORKSPACE')
+BUILD_NUMBER = os.environ.get('BUILD_NUMBER')
+BUILD_ID = os.environ.get('BUILD_ID')
+BUILD_URL = os.environ.get('BUILD_URL')
+
+GIT_REPO_PATH = os.environ.get('GIT_REPO_PATH')
+GIT_REPO_NAME = os.path.basename(GIT_REPO_PATH)
+GIT_OLD_ID = os.environ.get('GIT_OLD_ID')
+GIT_NEW_ID = os.environ.get('GIT_NEW_ID')
+GIT_BRANCH_NAME = os.environ.get('GIT_BRANCH_NAME')
+GIT_REMOTE_BRANCH_NAME = os.path.join('origin', '%s' %(GIT_BRANCH_NAME))
+GIT_TARGET_WORKSPACE = os.path.join(
+ CWD,
+ '%s-build%s' %(BUILD_ID, BUILD_NUMBER)
+)
+
+GIT_TARGET_DIR = os.path.join(
+ GIT_TARGET_WORKSPACE,
+ os.path.basename(GIT_REPO_PATH)
+)
+
+GIT_COMMITTER_EMAIL = os.environ.get('GIT_COMMITTER_EMAIL')
+
+KERNEL_CONFIG_PATH = os.path.join(GIT_TARGET_DIR, '.config')
+CREG_KERNEL_CONFIG_DELIM = re.compile(ur'\s*=\s*')
+PERSISTENCE_FILE = os.path.join(CWD, '..', '.persistence')
+SMTP_SERVER = 'roma.profitbricks.localdomain'
+SMTP_SUBJECT = 'Kernel build for branch %s, buildnumber %s was %s'
+SMTP_TEXT = (
+ 'Kernel build for branch %s, buildnumber %s was %s. ' +
+ 'Take a close look at: ' + BUILD_URL
+)
+SMTP_BUILD_SUCCESS = 'SUCCESSFULL'
+SMTP_BUILD_ERROR = 'NOT SUCCESSFULL'
+SMTP_FROM = 'hudson@profitbricks.com'
+PBUILDER = '/usr/sbin/pbuilder'
+SUDO = '/usr/bin/sudo'
+
+logger = logging.getLogger(sys.argv[0])
+logger.setLevel(logging.DEBUG)
+stream_handler = logging.StreamHandler()
+stream_handler.setLevel(logging.DEBUG)
+formatter = Formatter('%(asctime)s %(name)s[%(process)d] %(levelname)s: %(message)s')
+stream_handler.setFormatter(formatter)
+logger.addHandler(stream_handler)
+
+Base = declarative_base()
+
+engine = create_engine('sqlite:///%s' %(PERSISTENCE_FILE))
+Session = sessionmaker(bind=engine)
+
+def send_email(result):
+ smtp = smtplib.SMTP(SMTP_SERVER)
+ msg = (
+ 'From: %s\n' %(SMTP_FROM) +
+ 'To: %s\n' %(GIT_COMMITTER_EMAIL) +
+ 'Subject: %s\n' %(SMTP_SUBJECT %(GIT_BRANCH_NAME, BUILD_NUMBER,
+ result)) +
+ '%s\n' %(SMTP_TEXT %(GIT_BRANCH_NAME, BUILD_NUMBER, result))
+ )
+ smtp.sendmail(SMTP_FROM, GIT_COMMITTER_EMAIL, msg)
+ smtp.quit()
+
+class KernelVersion(Base):
+ __tablename__ = 'branch_kernel_version'
+ id = Column(Integer, Sequence('branch_id'), primary_key=True)
+ branch_name = Column(String(255), unique=True, nullable=False)
+ last_version = Column(Integer, nullable=False)
+
+ def __init__(self, branch_name, last_version):
+ self.branch_name = branch_name
+ self.last_version = last_version
+
+ def __repr__(self):
+ return '<KernelVersion(branch_name=\'%s\', last_version=\'%s\')>' %(
+ self.branch_name, self.last_version
+ )
+
+Base.metadata.create_all(engine)
+
+def get_last_kernel_revision_obj():
+ session = Session()
+ logger.debug('Getting kernel revision from persistence')
+ try:
+ answ = session.query(KernelVersion).filter(
+ KernelVersion.branch_name == GIT_BRANCH_NAME).one()
+ except NoResultFound:
+ session.add(KernelVersion(GIT_BRANCH_NAME, 0))
+ session.commit()
+ answ = session.query(KernelVersion).filter(
+ KernelVersion.branch_name == GIT_BRANCH_NAME).one()
+ finally:
+ session.close()
+
+ logger.info(
+ 'Got this Kernel revision for branch %s: %s'
+ %(GIT_BRANCH_NAME, answ)
+ )
+ return answ
+
+def update_kernel_revision_obj(kernel_version_obj, new_revision):
+ session = Session()
+ kernel_version_obj.last_version = new_revision
+ session.add(kernel_version_obj)
+ try:
+ session.commit()
+ except Exception, error:
+ logger.error('Some error happend while commiting new revision')
+ logger.exception(error)
+ return False
+ else:
+ return True
+ finally:
+ session.close()
+
+
+def build_kernel_with_pbuilder(revision, pbuilder_script, dist='stable'):
+ pbuilder_script = os.path.abspath(pbuilder_script)
+ cmd = [
+ SUDO,
+ PBUILDER,
+ '--execute',
+ '--hookdir', '""',
+ '--',
+ pbuilder_script,
+ '--cwd', '%s' %('/tmp'),
+ '--build-number', '%s' %(BUILD_NUMBER),
+ '--build-id', '%s' %(BUILD_ID),
+ '--build-url', '%s' %(BUILD_URL),
+ '--git-repo-path', '%s' %(GIT_REPO_PATH),
+ '--git-repo-name', '%s' %(GIT_REPO_NAME),
+ '--git-old-id', '%s' %(GIT_OLD_ID),
+ '--git-new-id', '%s' %(GIT_NEW_ID),
+ '--git-branch-name', '%s' %(GIT_BRANCH_NAME),
+ '--revision', '%s' %(revision),
+ '--dist', '%s' %(dist),
+ ]
+
+ os.environ.update({'DIST': '%s' %(dist)})
+
+ cmdobj = subprocess.Popen(
+ cmd,
+ shell=False,
+ close_fds=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ env=os.environ
+ )
+
+ logger.info('calling "%s" ...', ' '.join(cmd))
+# ret = cmdobj.wait()
+ ret = os.system('%s' %(' '.join(cmd)))
+ if ret:
+# errormsg = 'stdout: %s\n\n\nstderr: %s' %(cmdobj.stdout.read(),
+# cmdobj.stderr.read())
+# if not errormsg:
+# errormsg = None
+ logger.error(
+ '"%s" returned non-zero (exitcode was: %s).',
+ ' '.join(cmd),
+ ret,
+ )
+ return False
+
+# message = cmdobj.stdout.read()
+# if not message:
+# message = None
+ logger.info(
+ '"%s" returned zero.', ' '.join(cmd))
+ return True
+
+def read_file(path):
+ try:
+ fh = open(path, 'r', 1)
+ except:
+ raise
+ else:
+ result = dict(enumerate(fh))
+ fh.close()
+ return result
+
+def remove_git_target_workspace():
+ try:
+ shutil.rmtree(GIT_TARGET_WORKSPACE)
+ except IOError, error:
+ logger.debug('Got exception with error code: %s', error.errno)
+ if error.errno == 2:
+ pass
+ else:
+ raise
+ logger.info('deleted %s' %(GIT_TARGET_WORKSPACE))
+
+def exit_ok():
+ send_email(SMTP_BUILD_SUCCESS)
+ sys.exit(0)
+
+def exit_error():
+ send_email(SMTP_BUILD_ERROR)
+ sys.exit(1)
+
+if __name__ == '__main__':
+ logger.debug('running with this enviroment: %s', os.environ)
+ atexit.register(remove_git_target_workspace)
+ kernel_revision_obj = get_last_kernel_revision_obj()
+ new_version = kernel_revision_obj.last_version + 1
+ if not update_kernel_revision_obj(kernel_revision_obj, new_version):
+ logger.info(
+ 'Could not update persistence version to %s for %s'
+ %(new_version, GIT_BRANCH_NAME)
+ )
+ exit_error()
+
+ if not build_kernel_with_pbuilder(
+ new_version, 'kernel_inpbuilder.py',
+ dist='stable'):
+ exit_error()
+ exit_ok()
--- /dev/null
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+import re
+import sys
+import git
+import atexit
+import shutil
+import subprocess
+import logging
+import platform
+import smtplib
+import optparse
+from glob import glob
+from logging import Formatter
+#from ftplib import FTP
+from multiprocessing import cpu_count
+
+__version__ = '0.0.1'
+
+GIT = '/usr/bin/git'
+MAKE_KPKG = '/usr/bin/make-kpkg'
+APT_GET = '/usr/bin/apt-get'
+
+DEFAULT_PARALLEL_JOBS = cpu_count() + 1
+
+BUILD_ARCH_MAP = {
+ 'x86_64': 'amd64',
+ 'i386': '686'
+}
+
+BUILD_ARCH = BUILD_ARCH_MAP.get(platform.machine(), '686')
+CREG_KERNEL_CONFIG_DELIM = re.compile(ur'\s*=\s*')
+ERROR = 1
+OK = 0
+
+def getopts():
+ usage = '%prog [options]'
+ parser = optparse.OptionParser(
+ usage=usage,
+ version='%prog ' + __version__
+ )
+
+ parser.add_option(
+ '--cwd',
+ dest='cwd',
+ default='/tmp',
+ metavar='CWD',
+ help='Set the current working dir. Default: %default'
+ )
+
+ parser.add_option(
+ '--build-number',
+ dest='build_number',
+ default=None,
+ metavar='BUILD_NUMBER',
+ help='Set the Build number. Default: %default'
+ )
+
+ parser.add_option(
+ '--build-id',
+ dest='build_id',
+ default=None,
+ metavar='BUILD_ID',
+ help='Set the Build Id. Default: %default'
+ )
+
+ parser.add_option(
+ '--build-url',
+ dest='build_url',
+ default=None,
+ metavar='BUILD_URL',
+ help='Set the Build Url. Default: %default'
+ )
+
+ parser.add_option(
+ '--git-repo-path',
+ dest='git_repo_path',
+ default=None,
+ metavar='GIT_REPO_PATH',
+ help='Set the Git Repo Path. Default: %default'
+ )
+
+ parser.add_option(
+ '--git-repo-name',
+ dest='git_repo_name',
+ default=None,
+ metavar='GIT_REPO_NAME',
+ help='Set the Git Repo Name. Default: %default'
+ )
+
+ parser.add_option(
+ '--git-old-id',
+ dest='git_old_id',
+ default=None,
+ metavar='GIT_OLD_ID',
+ help='Set the Git Old Id. Default: %default'
+ )
+
+ parser.add_option(
+ '--git-new-id',
+ dest='git_new_id',
+ default=None,
+ metavar='GIT_NEW_ID',
+ help='Set the Git New Id. Default: %default'
+ )
+
+ parser.add_option(
+ '--git-branch-name',
+ dest='git_branch_name',
+ default=None,
+ metavar='GIT_BRANCH_NAME',
+ help='Set the Name of the branch. Default: %default'
+ )
+
+ parser.add_option(
+ '--revision',
+ dest='revision',
+ default=None,
+ metavar='REVISION',
+ help='Set the Revision. Default: %default'
+ )
+
+ parser.add_option(
+ '--dist',
+ dest='dist',
+ default='stable',
+ metavar='DIST',
+ help='Set the distribution. Default: %default'
+ )
+
+ return parser.parse_args()
+
+def git_clone_remote_repository(url, destination):
+ if os.path.exists(destination):
+ logger.debug('%s allready exists' %(destination))
+ if os.path.isdir(destination):
+ shutil.rmtree(destination)
+ else:
+ os.unlink(destination)
+ logger.debug('%s deleted' %(destination))
+
+ cmd = [GIT, 'clone', '%s' %(url), '%s' %(destination)]
+ cmdobj = subprocess.Popen(
+ cmd,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ shell=False,
+ env={'':''},
+ cwd=CWD,
+ close_fds=True
+ )
+
+ logger.info('begin to clone git repo from %s' %(url))
+ logger.debug(
+ 'calling »%s« for cloning git repo' %(' '.join(cmd))
+ )
+ ret = cmdobj.wait()
+
+ if ret:
+ logger.error('%s returned with %s' %(' '.join(cmd), ret))
+ logger.error('Error was: %s' %(cmdobj.stderr.readlines()))
+ return False
+ logger.debug('repository %s checked out into %s' %(url, destination))
+ return True
+
+def git_checkout_branch():
+ cmd = [GIT, 'checkout', '-b', GIT_BRANCH_NAME, GIT_REMOTE_BRANCH_NAME]
+
+ cmdobj = subprocess.Popen(
+ cmd,
+ shell=False,
+ close_fds=True,
+ stderr=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ env={'':''},
+ cwd=GIT_TARGET_DIR
+ )
+
+ logger.info(
+ 'checking out local branch %s from remote branch %s'
+ %(GIT_BRANCH_NAME, GIT_REMOTE_BRANCH_NAME)
+ )
+
+ logger.debug(
+ 'calling »%s« for checkout' %(' '.join(cmd))
+ )
+
+ ret = cmdobj.wait()
+ if ret:
+ logger.error('%s returned with %s' %(' '.join(cmd), ret))
+ logger.error('Error was: %s' %(cmdobj.stderr.readlines()))
+ return False
+ logger.info(
+ 'local branch %s successfully checked out.' %(GIT_BRANCH_NAME)
+ )
+ return True
+
+def build_kernel(revision, parallel_jobs='auto', distcc=False):
+ if parallel_jobs == 'auto':
+ parallel_jobs = DEFAULT_PARALLEL_JOBS
+ else:
+ parallel_jobs = int(parallel_jobs)
+
+ cmd = [MAKE_KPKG, '-j', '%s' %(parallel_jobs), '--arch',
+ '%s' %(BUILD_ARCH), '--rootcmd', 'fakeroot', '--revision',
+ '%s' %(revision), '--initrd', '--arch-in-name', 'kernel_debug',
+ 'kernel_image', 'kernel_source', 'kernel_headers', 'modules']
+
+ logger.info('start compile process')
+
+ logger.debug(
+ 'calling "%s" for compiling the kernel' %(' '.join(cmd))
+ )
+
+ os.putenv('LOCALVERSION', '')
+ os.environ['LOCALVERSION'] = ''
+ if distcc:
+ os.environ['MAKEFLAGS'] = 'CC=distcc'
+ os.putenv('MAKEFLAGS', 'CC=distcc')
+
+ ret = os.system('%s' %(' '.join(cmd)))
+
+ if ret:
+ logger.error('%s returned with %s' %(' '.join(cmd), ret))
+ return False
+ logger.info(
+ 'kernel successfully compiled.'
+ )
+ return True
+
+def read_file(path):
+ try:
+ fh = open(path, 'r', 1)
+ except:
+ raise
+ else:
+ result = dict(enumerate(fh))
+ fh.close()
+ return result
+
+def add_local_version_to_config(kernel_build_revision):
+ if not os.path.exists(KERNEL_CONFIG_PATH):
+ raise Exception(
+ 'No config file found at %s' %(KERNEL_CONFIG_PATH)
+ )
+ kernel_config = read_file(KERNEL_CONFIG_PATH)
+ localversion = '-%s-%s' %(GIT_BRANCH_NAME, BUILD_ARCH)
+ for lino, line in kernel_config.iteritems():
+ try:
+ key, value = CREG_KERNEL_CONFIG_DELIM.split(line)
+ except ValueError:
+ # found a comment line or something else
+ # we dont want.
+ continue
+ if key == 'CONFIG_LOCALVERSION':
+ value = localversion
+ kernel_config.update(((lino, '%s="%s"\n' %(key, value)),))
+ break
+ else:
+ # no CONFIG_LOCALVERSION in kernel config
+ lino += 1
+ kernel_config.setdefault(
+ lino,
+ 'CONFIG_LOCALVERSION="%s"\n' %(localversion)
+ )
+
+ try:
+ fh = open(KERNEL_CONFIG_PATH, 'w', 1)
+ except:
+ raise
+ else:
+ for lino, line in sorted(
+ kernel_config.iteritems(),
+ key=lambda x: x[0]
+ ):
+ fh.write(line)
+ fh.close()
+ return localversion
+ return False
+
+def remove_git_target_workspace():
+ try:
+ shutil.rmtree(GIT_TARGET_WORKSPACE)
+ except IOError, error:
+ if error.errno == 2:
+ pass
+ else:
+ raise
+ logger.info('deleted %s' %(GIT_TARGET_WORKSPACE))
+
+def exit(retcode):
+ logger.info('Exit with %s', retcode)
+ sys.exit(retcode)
+
+
+def has_extra_modules():
+ return os.path.exists(EXTRA_MODULES_PATH)
+
+def install_extra_modules():
+ cmd = [APT_GET, 'update']
+ subprocess.check_call(cmd)
+
+ fh = open(EXTRA_MODULES_PATH, 'r', 1)
+ extra_modules = map(lambda x: x.rstrip(), fh)
+ fh.close()
+
+ if extra_modules:
+ cmd = [APT_GET, 'install']
+ cmd += extra_modules
+ subprocess.check_call(cmd)
+ return True
+
+
+if __name__ == '__main__':
+ logger = logging.getLogger(sys.argv[0])
+ logger.setLevel(logging.DEBUG)
+ stream_handler = logging.StreamHandler(sys.stdout)
+ stream_handler.setLevel(logging.DEBUG)
+ formatter = Formatter('%(asctime)s %(name)s[%(process)d] %(levelname)s: %(message)s')
+ stream_handler.setFormatter(formatter)
+ logger.addHandler(stream_handler)
+
+ options, args = getopts()
+ logger.debug('running with this enviroment: %s', os.environ)
+
+ CWD = os.environ.get('WORKSPACE') or options.cwd
+ if not CWD:
+ logger.info('No CWD given - using "/tmp" as CWD')
+ CWD = '/tmp'
+
+ BUILD_NUMBER = os.environ.get('BUILD_NUMBER') or options.build_number
+ if not BUILD_NUMBER:
+ logger.error('No build number given')
+ exit(ERROR)
+
+ BUILD_ID = os.environ.get('BUILD_ID') or options.build_id
+ if not BUILD_ID:
+ logger.error('No build id given')
+ exit(ERROR)
+
+ # FIXME: Do we realy need BUILD_URL here?
+ BUILD_URL = os.environ.get('BUILD_URL') or options.build_url
+ if not BUILD_URL:
+ logger.error('No build url given')
+ exit(ERROR)
+
+ GIT_REPO_PATH = os.environ.get('GIT_REPO_PATH') or options.git_repo_path
+ if not GIT_REPO_PATH:
+ logger.error('No git repo path given')
+ exit(ERROR)
+
+ GIT_OLD_ID = os.environ.get('GIT_OLD_ID') or options.git_old_id
+ if not GIT_OLD_ID:
+ logger.error('No git old id given')
+ exit(ERROR)
+
+ GIT_NEW_ID = os.environ.get('GIT_NEW_ID') or options.git_new_id
+ if not GIT_NEW_ID:
+ logger.error('No git new id given')
+ exit(ERROR)
+
+ GIT_BRANCH_NAME = (
+ os.environ.get('GIT_BRANCH_NAME') or options.git_branch_name
+ )
+ if not GIT_BRANCH_NAME:
+ logger.error('No git branch name given')
+ exit(ERROR)
+
+ REVISION = os.environ.get('REVISION') or options.revision
+ if not REVISION:
+ logger.error('No revision given')
+ exit(ERROR)
+
+ GIT_REPO_NAME = os.path.basename(GIT_REPO_PATH)
+ GIT_REMOTE_BRANCH_NAME = os.path.join('origin', '%s' %(GIT_BRANCH_NAME))
+ GIT_TARGET_WORKSPACE = os.path.join(
+ CWD,
+ '%s-build%s' %(BUILD_ID, BUILD_NUMBER)
+ )
+
+ GIT_TARGET_DIR = os.path.join(
+ GIT_TARGET_WORKSPACE,
+ os.path.basename(GIT_REPO_PATH)
+ )
+
+ GIT_COMMITTER_EMAIL = os.environ.get('GIT_COMMITTER_EMAIL')
+
+ KERNEL_CONFIG_PATH = os.path.join(GIT_TARGET_DIR, '.config')
+
+ EXTRA_MODULES_PATH = os.path.join(GIT_TARGET_DIR, '.modules')
+
+ if git_clone_remote_repository(GIT_REPO_PATH, GIT_TARGET_DIR):
+ logger.info('git clone was successfull')
+ else:
+ logger.info('git clone was not successfull')
+ exit(ERROR)
+ #atexit.register(remove_git_target_workspace)
+ if not git_checkout_branch():
+ exit(ERROR)
+ #kernel_revision_obj = get_last_kernel_revision_obj()
+ #new_version = kernel_revision_obj.last_version + 1
+ new_version = REVISION
+ try:
+ add_local_version_to_config(new_version)
+ except Exception, error:
+ logger.exception(error)
+ exit(ERROR)
+ else:
+ logger.info(
+ 'updated CONFIG_LOCALVERSION in %s to %s'
+ %(KERNEL_CONFIG_PATH, new_version)
+ )
+# if not update_kernel_revision_obj(kernel_revision_obj, new_version):
+# logger.info(
+# 'Could not update persistence version to %s for %s'
+# %(new_version, GIT_BRANCH_NAME)
+# )
+# exit(ERROR)
+
+ logger.debug('changing dir to %s' %(GIT_TARGET_DIR))
+ os.chdir(GIT_TARGET_DIR)
+
+ if has_extra_modules():
+ if not install_extra_modules():
+ logger.error('Error during installation of extra kernel modules')
+ exit(ERROR)
+
+ if not build_kernel(new_version):
+ exit(ERROR)
+ else:
+ #ftp = FTP(
+ # 'alexandria.profitbricks.localdomain',
+ # 'debian-uploader',
+ # 'vae6tooZe1ec'
+ #)
+
+ #logger.info('FTP Login on %s successfull' %(ftp.host))
+
+ #ftp.cwd('squeeze')
+ #for package in glob(
+ # os.path.join(GIT_TARGET_WORKSPACE, '*.deb')
+ #):
+ # fh = open(package, 'rb', 1)
+ # ftp.storbinary(
+ # 'STOR %s' %(os.path.basename(package)),
+ # fh
+ # )
+ # fh.close()
+ # logger.info('Successfully uploaded %s' %(package))
+ #ftp.quit()
+ logger.info('Build successfull')
+ logger.info('dirlist: %s' %(os.listdir(GIT_TARGET_WORKSPACE)))
+ exit(OK)
projects / profitbricks / jenkins-build-scripts.git / commitdiff