From: Benjamin Drung Date: Thu, 16 Jan 2014 18:59:23 +0000 (+0100) Subject: Add initial version of gentoo_build. X-Git-Url: https://git.uhu-banane.net/?a=commitdiff_plain;h=27b4c7a45c0cee66cdf2063d490989ea39bfeae7;p=profitbricks%2Fjenkins-build-scripts.git Add initial version of gentoo_build. --- diff --git a/gentoo_build b/gentoo_build new file mode 100755 index 0000000..e546600 --- /dev/null +++ b/gentoo_build @@ -0,0 +1,382 @@ +#!/usr/bin/python + +# Copyright (C) 2014, ProfitBricks GmbH +# Author: Benjamin Drung +# +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +"""Do a source release for Gentoo (release a versioned ebuild file)""" + +import glob +import logging +import optparse +import os +import re +import shutil +import sys +import subprocess +import tempfile + +# We need python-git >= 0.3 +import debian.changelog +import git + +import common_code + +DEFAULT_CHROOT_NAME = "gentoo" +DEFAULT_CHROOT_USER = "root" + +ENV_CATEGORY = "CATEGORY" +ENV_CHROOT = "CHROOT_NAME" +ENV_CHROOT_USER = "CHROOT_USER" +ENV_RELEASE_BRANCH = "RELEASE_BRANCH" +ENV_RELEASE_REPO = "RELEASE_REPO" +ENV_VARIABLES = [ + ENV_CATEGORY, + ENV_CHROOT, + ENV_CHROOT_USER, + ENV_RELEASE_BRANCH, + ENV_RELEASE_REPO, +] + +CATEGORY_VARIABLE = "PB_CATEGORY" +CATEGORY_RE = r'^\s*' + CATEGORY_VARIABLE + r'=\s*"?([^"]*)"?\s*$' + +class Ebuild(object): + """This class represents an .ebuild file.""" + + def __init__(self, logger, filename, category=None): + assert os.path.isfile(filename) + self.logger = logger + # The file should be named .ebuild + basename = os.path.basename(filename) + self.name = os.path.splitext(basename)[0] + self.version = None + self.ebuild = open(filename).read() + self.category = category + if self.category is None: + # Read PB_CATEGORY from .ebuild file + match = re.search(CATEGORY_RE, self.ebuild, re.MULTILINE) + if match: + self.category = match.group(1).strip() + else: + logger.error("No category was specified.\n" + "Please provide one on the command line, set " + "{env} in the environment, or set {var} in " + "the {file} file.".format(env=ENV_CATEGORY, + var=CATEGORY_VARIABLE, + file=basename)) + sys.exit(1) + + def get_relpath(self): + """Return the relative path of the ebuild file. + The path is //-.ebuild + """ + assert self.category is not None + assert self.version is not None + filename = self.name + "-" + self.version + ".ebuild" + return os.path.join(self.category, self.name, filename) + + def save(self, base_path): + """Save the ebuild file below the given base_path.""" + fullpath = os.path.join(base_path, self.get_relpath()) + ebuild_file = open(fullpath, "w") + ebuild_file.write(self.ebuild) + ebuild_file.close() + self.logger.info("Ebuild file saved as " + self.get_relpath()) + + def set_git_commit(self, commit): + """Set EGIT_COMMIT in the ebuild file to the given commit.""" + (self.ebuild, n_subs) = re_subn(r'^\s*EGIT_COMMIT=.*$', + 'EGIT_COMMIT="' + commit + '"', + self.ebuild, flags=re.MULTILINE) + if n_subs == 0: + self.logger.warn("No EGIT_COMMIT variable found in ebuild file!") + elif n_subs > 1: + self.logger.warn("Replaced {n} occurrences of the EGIT_COMMIT " + "variable in the ebuild file, but expected only " + "exactly one!".format(n=n_subs)) + + def set_version(self, version): + """Specify the version of the ebuild file.""" + self.version = debian2gentoo_version(version) + self.logger.info('Set version of {name} ebuild to ' + '"{version}".'.format(name=self.name, + version=self.version)) + + +def debian2gentoo_version(debian_version): + """Translate a Debian version into a Gentoo version. + + We just replace dashes and pluses (from the Debian revision) by points. + More suffisticated version mapping from Gentoo to Debian is not done. + """ + return debian_version.replace("-", ".") + +def re_subn(pattern, repl, string, count=0, flags=0): + """Implement multi-line string replacement. + This function is a work-around for the missing flags parameter + in Python <= 2.6. You can replace it by re.subn in Python >= 2.7 + """ + assert flags == re.MULTILINE + if sys.version_info < (2, 7, 0): + sum_number_of_subs_made = 0 + lines = string.split("\n") + for i in range(len(lines)): + (lines[i], n_subs_made) = re.subn(pattern, repl, lines[i], count) + sum_number_of_subs_made += n_subs_made + return ("\n".join(lines), sum_number_of_subs_made) + else: + return re.subn(pattern, repl, string, count, flags) + +def main(): + """Read command-line parameters and needed environment variables.""" + script_name = os.path.basename(sys.argv[0]) + usage = "%s [options] [release repository]" % (script_name) + epilog = "Supported environment variables: " + ", ".join(ENV_VARIABLES) + parser = optparse.OptionParser(usage=usage, epilog=epilog) + + env = os.environ + used_env = dict((k, v) for (k, v) in env.items() if k in ENV_VARIABLES) + env.setdefault(ENV_CHROOT, DEFAULT_CHROOT_NAME) + env.setdefault(ENV_CHROOT_USER, DEFAULT_CHROOT_USER) + + log_levels = { + 'DEBUG': logging.DEBUG, + 'INFO': logging.INFO, + 'WARN': logging.WARN, + 'WARNING': logging.WARNING, + 'ERROR': logging.ERROR, + 'CRITICAL': logging.CRITICAL + } + + parser.add_option("-b", "--branch", dest="release_branch", + default=env.get(ENV_RELEASE_BRANCH), + help="specify a branch for the release repository " + "(default: same branch as the source repository)") + parser.add_option("-c", "--category", dest="category", + default=env.get(ENV_CATEGORY), + help="specify a category for the ebuild file") + parser.add_option("--chroot", dest="chroot_name", + default=env[ENV_CHROOT], + help="chroot to use for calling ebuild (default: " + "{default})".format(default = DEFAULT_CHROOT_NAME)) + parser.add_option("--chroot-user", dest="chroot_user", + default=env[ENV_CHROOT_USER], + help="chroot user to use for calling ebuild (default: " + "{default})".format(default = DEFAULT_CHROOT_USER)) + parser.add_option("-d", "--directory", dest="directory", + help="change to the specified directory") + parser.add_option("-n", "--dry-run", dest="dry_run", action="store_true", + help="perform a trial run with no changes made") + parser.add_option("--keep", dest="keep", action="store_true", + help="keep the temporary cloned directory of the " + "release repository") + parser.add_option("--tag-from-debian-changelog", dest="tag_from_debian", + action="store_true", help="tag head commit with version " + "from debian/changelog") + parser.add_option('--loglevel', dest='loglevel', choices=log_levels.keys(), + default='INFO', help='Loglevel. Default: %default') + + (options, args) = parser.parse_args() + + logger = common_code.logger_init(script_name, + log_level=log_levels[options.loglevel]) + logger.debug("Command line arguments: " + " ".join(sys.argv)) + logger.debug('Start-up environment:\n\n{env}'.format(env=shell_env())) + logger.info('Used environment variables: {env}'.format(env=used_env)) + + if len(args) > 1: + parser.error("This script does take at most one additional parameter.") + + if len(args) >= 1: + release_repo = args[0] + elif ENV_RELEASE_REPO in env: + release_repo = env[ENV_RELEASE_REPO] + else: + parser.error("No release repository was specified.\n" + "Please provide one on the command line or set {env} in " + "the environment.".format(env=ENV_RELEASE_REPO)) + + if options.directory: + os.chdir(options.directory) + + gentoo_build(script_name, logger, release_repo, options) + +def shell_env(): + """Return all environment variables in a format that can be parsed by + a shell again.""" + env = "" + for key, value in os.environ.iteritems(): + env += '{key}="{value}"\n'.format(key=key, value=value) + return env + +def gentoo_build(script_name, logger, release_repo_uri, options): + """Main function that creates a versioned ebuild file and releases it.""" + ebuild = find_ebuild(logger, options.category) + if options.tag_from_debian: + git_tag = tag_from_debian_changelog(logger) + else: + git_tag = get_latest_tag(logger) + ebuild.set_version(git_tag) + ebuild.set_git_commit(git_tag) + tmpdir = tempfile.mkdtemp(prefix=script_name+".") + try: + # Checkout release repository + branch = options.release_branch + if branch is None: + branch = git.Repo().active_branch.name + logger.info("Check out {branch} branch of {repo} to " + "{dir}...".format(repo=release_repo_uri, branch=branch, + dir=tmpdir)) + check_branch_existance(logger, release_repo_uri, branch) + release_repo = git.Repo.clone_from(release_repo_uri, tmpdir, b=branch) + # Copy modified ebuild into release repository + ebuild.save(tmpdir) + # Update manifest file (in a Gentoo chroot) + cmd = ["schroot", "-c", options.chroot_name, "-u", options.chroot_user, + "-d", tmpdir, "--", "ebuild", ebuild.get_relpath(), "manifest"] + logger.info("Calling " + " ".join(cmd)) + retcode = subprocess.call(cmd) + if retcode != 0: + logger.error("ebuild command in Gentoo schroot failed with exit " + "code {code}.".format(code=retcode)) + sys.exit(retcode) + release_repo.git.add("-A") + release_repo.git.commit("-m", "Added " + ebuild.get_relpath()) + logger.info("Adding following commit:\n" + \ + release_repo.git.log("-p", "HEAD~1..HEAD")) + # Push changes to the release repository + if options.dry_run: + logger.info("Not pushing changes to {repo} as dry run was " + "requested.".format(repo=release_repo_uri)) + else: + logger.info("Pushing {branch} branch to {repo}" + "...".format(branch=branch, repo=release_repo_uri)) + release_repo.git.push("origin", branch) + + if options.tag_from_debian: + push_tag(logger, git_tag, options.dry_run) + finally: + if options.keep: + logger.warning("Keeping temporary git clone in {dir} as " + "requested.".format(dir=tmpdir)) + else: + shutil.rmtree(tmpdir) + +def check_branch_existance(logger, repo_uri, branch): + """Check if the specified remote repository has the given branch. + This check is needed, because a git clone on Debian 6.0 (squeeze) + will checkout the master branch if the specified version does not + exist. + """ + repo = git.Repo(repo_uri) + branches = [b.name for b in repo.branches] + if branch not in branches: + logger.error("Remote branch {branch} not found in " + "{repo}".format(branch=branch, repo=repo_uri)) + sys.exit(1) + +def find_ebuild(logger, category): + """Search for exactly one ebuild file in the current directory""" + ebuild_files = glob.glob("*.ebuild") + if len(ebuild_files) == 0: + logger.error('No .ebuild file found in "{dir}".\n' + 'Did you call the script from the correct directory and ' + 'does your project contain an .ebuild file in the top ' + 'directory?'.format(dir=os.getcwd())) + sys.exit(1) + if len(ebuild_files) > 1: + logger.error('More than one .ebuild file found in "{dir}": ' + '{ebuild_files}\nThe script expects exactly one .ebuild ' + 'file.'.format(dir=os.getcwd(), + ebuild_files=ebuild_files)) + sys.exit(1) + ebuild_file = ebuild_files[0] + logger.info('Use ebuild file: {ebuild}'.format(ebuild=ebuild_file)) + return Ebuild(logger, ebuild_file, category) + +def tag_from_debian_changelog(logger): + """Create a tag from the version specified in debian/changelog. + Returns the name of the created tag. + """ + changelog_file = open('debian/changelog') + changelog = debian.changelog.Changelog(changelog_file, max_blocks=1) + version = changelog.full_version + # Replace valid Debian version chars that are invalid for git tagging + new_tag = version.replace('~', '_').replace(':', ',') + tag_head_commit(logger, new_tag) + return new_tag + +def tag_head_commit(logger, new_tag): + """Tags the head commit with the given tag name.""" + repo = git.Repo('.') + current_commit = repo.commit(repo.active_branch) + tags = [t.name for t in repo.tags if t.commit == current_commit] + if len(tags) > 0: + if len(tags) == 1 and tags[0] == new_tag: + msg = ('The head commit {commit} is already tagged with {tag}.\n' + 'Call this script without --tag-from-debian-changelog ' + 'to release the ebuild file.') + msg = msg.format(commit=current_commit.hexsha[0:7], tag=new_tag) + else: + msg = ('The head commit {commit} is already tagged: {tags}\n' + 'The script expects a commit without additional tags.') + msg = msg.format(commit=current_commit.hexsha[0:7], + tags=" ".join(tags)) + logger.error(msg) + sys.exit(1) + remote_tag = [t for t in repo.tags if t.name == new_tag] + if len(remote_tag) > 0: + msg = 'Tag {tag} was already created for commit {commit}.' + logger.error(msg.format(tag=new_tag, + commit=remote_tag[0].commit.hexsha[0:7])) + sys.exit(1) + logger.info("Tagging commit {commit} with {tag}" + "...".format(commit=current_commit.hexsha[0:7], tag=new_tag)) + repo.git.tag(new_tag) + +def get_latest_tag(logger): + """Get the tag name for the branch head commit. + The function will fail if the branch head commit is not tagged or has + multiple tags. + """ + repo = git.Repo('.') + current_commit = repo.commit(repo.active_branch) + tags = [t.name for t in repo.tags if t.commit == current_commit] + if len(tags) == 0: + logger.error('No tag found for commit {commit}.\nHave you tagged ' + 'your release?'.format(commit=current_commit.hexsha[0:7])) + sys.exit(1) + if len(tags) > 1: + msg = ('More than one tag found for commit {commit}: {tags}\n' + 'The script expects exactly one tag.') + logger.error(msg.format(commit=current_commit.hexsha[0:7], + tags=" ".join(tags))) + sys.exit(1) + tag = tags[0] + logger.info('Use tag "{tag}" as release version.'.format(tag=tag)) + return tag + +def push_tag(logger, tag, dry_run): + """Pushed the given git tag unless it is a dry run.""" + if dry_run: + logger.info("Not pushing tag {tag} as dry run was " + "requested.".format(tag=tag)) + else: + logger.info("Pushing tag {tag}...".format(tag=tag)) + repo = git.Repo() + repo.git.push("origin", tag) + +if __name__ == '__main__': + main() \ No newline at end of file