Logo Search packages:      
Sourcecode: falcon version File versions  Download package

build.py

# This file is part of the Falcon repository manager
# Copyright (C) 2005-2008 Dennis Kaarsemaker
# See the file named COPYING in the root of the source tree for license details
#
# build.py - Package build coordination functions

import falcon
from django.db import models
import os, random, gzip, tarfile, shutil, datetime
import cPickle as pickle
from email import FeedParser

BUILD_FAILED, BUILD_PENDING, BUILD_OK = range(3)

def build(dsc):
    """Main build function, takes a dsc filename (full path), performs checks
       and queues all builds"""
    falcon.util.debug(_("Parsing %s") % dsc)
    control, controlfields, files = falcon.package.parse_dsc(dsc)
    path = os.path.dirname(dsc)

    # Do we need arch-specific builders?
    archs = controlfields['Architecture'].replace(',',' ').split()
    found_debcontrol = False
    found_debchangelog = False 
    pocket = falcon.conf.pocket
    component = falcon.conf.component

    # Look for debian/control and debian/changelog in diff.gz
    for f in files:
        if f.name.endswith('diff.gz'):
            g = gzip.open(os.path.join(path,f.name))
            while True:
                l = g.readline()
                if not l:
                    break
                l = l.strip()
                if l.startswith('+++') and l.endswith('/debian/control'):
                    found_debcontrol = True
                elif l.startswith('+++') and l.endswith('/debian/changelog'):
                    found_debchangelog = True
                elif found_debcontrol and l.startswith('+Architecture'):
                    if not l.strip().endswith('all'):
                        need_arch_builds=True
                elif found_debcontrol and l.startswith('+Section') and not component:
                    component = l[l.find(':')+1:].strip()
                    if '/' in component:
                        component = component[:component.find('/')].strip()
                    else:
                        component = 'main'
                elif found_debchangelog and 'urgency' in l:
                    if not pocket:
                        pocket = l[l.find(')')+1:l.find(';')].strip()
                        pockets = [p.name for p in falcon.pocket.Pocket.objects.all()]
                        if pocket not in pockets:
                            for p in pockets:
                                if p.startswith(pocket):
                                    pocket = p
                                    break
                        
                elif found_debcontrol and found_debchangelog and l.startswith('+++'):
                    break
    if not found_debcontrol or not found_debchangelog:
        tf = None
        for f in files:
            if f.name.endswith('tar.gz'):
                tf = tarfile.open(name=os.path.join(path, f.name), mode='r:gz')
            elif f.name.endswith('tar.bz2'):
                tf = tarfile.open(name=os.path.join(path, f.name), mode='r:bz2')
            if tf:
                for n in tf.getnames():
                    if n.endswith('debian/control'):
                        found_debcontrol = True
                        control = tf.extractfile(n).readlines()
                        for l in control:
                            if l.startswith('Section') and not component:
                                component = l[l.find(':')+1:].strip()
                                if '/' in component:
                                    component = component[:component.find('/')].strip()
                                else:
                                    component = 'main'
                            if l.startswith('Architecture'):
                                if not l.strip().endswith('all'):
                                    need_arch_builds=True
                    if n.endswith('debian/changelog'):
                        found_debchangelog = True
                        if not falcon.conf.pocket:
                            l = tf.extractfile(n).readline()
                            pocket = l[l.find(')')+1:l.find(';')].strip()
                    if found_debcontrol and found_debchangelog:
                        break

    if not found_debcontrol:
        falcon.util.warning(_("Can't find debian/control"))
        return
    if not found_debchangelog:
        falcon.util.warning(_("Can't find debian/changelog"))
        return

    # Did we install this package already?
    try:
        objs = falcon.package.SourcePackage.objects.filter(component__pocket__name = pocket,
                                                           packagename = controlfields['Source'],
                                                           version = controlfields['Version'])
        if len(objs):
            falcon.util.warning(_("Package %s %s has already been installed") % (controlfields['Source'], controlfields['Version']))
            return
    except falcon.package.SourcePackage.DoesNotExist:
        pass

    # Do we have this pocket?
    if falcon.conf.install_built:
        try:
            falcon.pocket.Component.objects.get(name=component, pocket__name=pocket)
        except falcon.pocket.Component.DoesNotExist:
            falcon.util.warning(_("Can't install to nonexisting component %s/%s") % (pocket, component))
            return

    qi = QueueItem(dscfile = os.path.basename(dsc),
                   packagename = controlfields['Source'],
                   version = controlfields['Version'],
                   pocket = pocket,
                   component = component,
                   install = bool(falcon.conf.install_built),
                   files = files,
                   status = {})
    qi.save()

    # Find non-busy builddeamons
    builders = list(Builder.objects.all())
    random.shuffle(builders)

    if archs == ['any']:
        archs = falcon.conf.architectures
    busy = []
    for qi in QueueItem.objects.all():
        busy += [x.id for x in qi.active_builders.all()]
    # Find an arch_all buildd
    if archs == ['all']:
        for b in builders:
            if b.arch_all and b.id not in busy and (archs == ['all'] or b.arch in archs):
                qi.active_builders.add(b)
                if b.arch in archs:
                    archs.remove(b.arch)
                break
        else:
            falcon.util.warning(_("Can't find a non-busy arch_all builder"))
            qi.delete()
            return
    # Find builders for all archs
    for a in archs:
        if a == 'all':
            continue
        for b in builders:
            if b.arch == a and b.id not in busy:
                qi.active_builders.add(b)
                break
        else:
            falcon.util.warning(_("Can't find a non-busy %s builder") % a)
            qi.delete()
            return

    falcon.util.output(_("Building %s on %s") % (os.path.basename(dsc), ", ".join([b.name for b in qi.active_builders.all()])))
    # Move to our buildqueue
    buildroot = os.path.join('.falcon','build','building')
    if os.path.realpath(path) != os.path.realpath(buildroot):
        for f in files:
            if os.path.exists(os.path.join(buildroot,f.name)):
                falcon.util.warning(_("File %s already exists in the build root"))
                return
            if os.path.exists(os.path.join(path, f.name)):
                shutil.move(os.path.join(path,f.name), buildroot)
            elif not f.name.endswith('orig.tar.gz'):
                falcon.util.warning(_("File %s does not exist") % (os.path.join(path, f.name)))
                return
            else:
                # This was a changes-only upload, try and find the orig.tar.gz
                # in the correct pocket
                pkgs = falcon.package.SourcePackage.objects.filter(pocket__name = falcon.conf.pocket, 
                                                                   packagename = controlfields['Source'])
                for p in pkgs:
                    for file in p.files:
                        if f.name == file.name:
                            shutil.copy(os.path.join(), buildroot)
                        break
                    else:
                        continue
                    break
                else:
                    falcon.util.warning(_("Can't find %s for changes-only upload %s %s") % (f.name, controlfields['Source'], controlfields['Version']))
                    return

    # Queue the builds
    qi.run(True)

00196 class Builder(models.Model):
    """Abstraction of a build daemon"""
    name = models.CharField(maxlength=30,unique=True)
    arch = models.CharField(maxlength=10)
    arch_all = models.BooleanField(default=False)
    hostname = models.CharField(maxlength=100)
    uploadpath = models.CharField(maxlength=200)
    downloadpath = models.CharField(maxlength=200)
    buildcommand = models.PickleField(default=[])
    comment = models.TextField()

    def __init__(self, *args, **kwargs):
        super(Builder, self).__init__(*args, **kwargs)
        if type(self.buildcommand) == str:
            self.buildcommand = pickle.loads(self.buildcommand)

    def __str__(self):
        arch = self.arch
        if self.arch_all:
            arch += '/all'
        return 'Falcon %s builder %s' % (arch, self.name)

    def __repr__(self):
        arch = self.arch
        if self.arch_all:
            arch += '/all'
        return '<Falcon %s builder %s>' % (arch, self.name)

    @falcon.plugin.wrap_plugin
00225     def build(self, tobuild):
        """Build a QueueItem on a build daemon"""
        path = os.path.join('.falcon','build','building')
        try:
            falcon.util.output(_("Uploading %s %s for building on %s buildd %s") % (tobuild.packagename, tobuild.version, self.arch, self.name))
            for f in tobuild.files:
                falcon.util.run(['rsync', os.path.join(path, f.name), self.uploadpath], buffer=not falcon.conf.verbose) 
            if self.buildcommand:
                for f in ('FULLYBUILT', 'FAILEDTOBUILD'):
                    if os.path.exists(os.path.join(self.downloadpath, tobuild.log_template() % (self.arch, f) + '.gz')):
                        falcon.util.warning(_("Not building %s %s, a buildlog already exists") % (tobuild.packagename, tobuild.version))
                        return BUILD_FAILED
                buildcommand = [x.replace('%(pocket)', tobuild.pocket).replace('%(dscfile)', os.path.basename(tobuild.dscfile))
                                for x in self.buildcommand]
                falcon.util.output(_("Trying to build on %s") % self.name)
                log = os.path.join(falcon.conf.rootdir, '.falcon', 'build','logs', tobuild.log_template() % (self.arch, 'BUILDING'))
                falcon.util.output(_("Executing the build command, logging to %s") % os.path.basename(log))
                try:
                    falcon.util.run(['ssh', self.hostname] + buildcommand, buffer=False, outfile=log, err_is_out=True)
                except RuntimeError:
                    newlog = os.path.join(falcon.conf.rootdir, '.falcon', 'build','logs', tobuild.log_template() % (self.arch, 'FAILEDTOBUILD'))
                    os.rename(log, newlog)
                    falcon.util.run(['gzip', '-f', newlog])
                    raise # Will be caught a few lines on
                newlog = os.path.join(falcon.conf.rootdir, '.falcon', 'build','logs', tobuild.log_template() % (self.arch, 'FULLYBUILT'))
                os.rename(log, newlog)
                falcon.util.run(['gzip', '-f', newlog])
                return self.download(tobuild)
            else:
                return BUILD_PENDING
        except RuntimeError, e:
            falcon.util.debug_exception()
            return BUILD_FAILED
            
00259     def download(self, tobuild):
        """Check a QueueItem for completion and download it"""
        resdir = os.path.join('.falcon','build','building')
        logdir = os.path.join('.falcon','build','logs')
        falcon.util.output(_("Checking build status of %s %s on %s buildd %s") % (tobuild.packagename, tobuild.version, self.arch, self.name))
        # Grab the log first
        if not self.buildcommand:
            logfile = os.path.join(self.downloadpath, tobuild.log_template() % (self.arch, 'FULLYBUILT')) + '.gz'
            dllog = os.path.join(resdir, logfile)
            try:
                falcon.util.run(['rsync', logfile, logdir], buffer=not falcon.conf.verbose)
            except RuntimeError:
                logfile = os.path.join(self.downloadpath, tobuild.log_template() % (self.arch, 'BUILDING')) + '.gz'
                try:
                    falcon.util.run(['rsync', logfile, logdir], buffer=not falcon.conf.verbose)
                    return BUILD_PENDING
                except RuntimeError:
                    falcon.util.debug(_("Couldn't find BUILDING log"))
                logfile = os.path.join(self.downloadpath, tobuild.log_template() % (self.arch, 'FAILEDTOBUILD')) + '.gz'
                try:
                    falcon.util.run(['rsync', logfile, logdir], buffer=not falcon.conf.verbose)
                    return BUILD_FAILED
                except RuntimeError:
                    falcon.util.debug(_("Couldn't find FAILEDTOBUILD log"))
                    # Assume still pending
                    return BUILD_PENDING

        # Changes file
        logfile = tobuild.changes_file() % self.arch
        if os.path.exists(os.path.join(resdir, logfile)):
            falcon.util.warning(_("Changes file %s already exists, not redownloading from buildd %s") % (logfile, self.name))
            return BUILD_FAILED
        try:
            falcon.util.run(['rsync', os.path.join(self.downloadpath, logfile), resdir], buffer=not falcon.conf.verbose)
        except RuntimeError, e:
            falcon.util.debug_exception()
            falcon.util.warning(_("Failed to download file %s from buildd %s") % (logfile, self.name))
            return BUILD_FAILED

        # Now see which files we can grab!
        log = os.path.join(resdir, logfile)
        fd = open(log)
        log = fd.read()
        fd.close()
        parser = FeedParser.FeedParser()
        parser.feed(log)
        log = parser.close()
        files = log['Files']
        for f in files.split("\n"):
            try:
                f = f.split()[-1]
                falcon.util.output(_("Downloading %s") % f)
                falcon.util.run(['rsync', os.path.join(self.downloadpath, f), resdir], buffer=not falcon.conf.verbose)
                # Since pkg_create_dbgsym doesn't add the .ddeb files to the .changes
                # file, try and grab one manually if it exists
                if f.endswith('.deb') and not f.endswith('all.deb'):
                    f2 = f[:-3].replace('_','-dbgsym_',1) + 'ddeb'
                    try:
                        falcon.util.output(_("Checking for debug symbol package %s") % f2)
                        falcon.util.run(['rsync', os.path.join(self.downloadpath, f2), resdir], buffer=not falcon.conf.verbose)
                    except RuntimeError, e:
                        falcon.util.output(_("Not found"))
            except RuntimeError, e:
                falcon.util.debug_exception()
                falcon.util.warning(_("Failed to download file %s from buildd %s") % (f, self.name))
                return BUILD_FAILED
        return BUILD_OK

00327 class QueueItem(models.Model):
    """Abstraction of a queued package and build state"""
    # .dsc or .changes file
    dscfile = models.CharField(maxlength=100)
    files = models.PickleField()
    packagename = models.CharField(maxlength=50)
    version = models.CharField(maxlength=15)
    pocket = models.CharField(maxlength=20)
    component = models.CharField(maxlength=20)
    install = models.BooleanField(default=False)
    # Used builders
    active_builders = models.ManyToManyField(Builder, related_name='active_packages')
    inactive_builders = models.ManyToManyField(Builder, related_name='inactive_packages')
    status = models.PickleField()

    def __init__(self, *args, **kwargs):
        super(QueueItem, self).__init__(*args, **kwargs)
        if type(self.files) == str:
            self.files = pickle.loads(self.files)
        if type(self.status) == str:
            self.status = pickle.loads(self.status)

    def log_template(self):
        pname = self.pocket
        if '-' in pname:
            pname = pname[:pname.find('-')]
        return 'buildlog_%s-%s-%%s.%s_%s_%%s.txt' % (falcon.conf.origin, pname, self.packagename, self.version)

    def changes_file(self):
        version = self.version
        if ':' in version:
            version = version[version.find(':')+1:]
        return '%s_%s_%%s.changes' % (self.packagename, version)

    @falcon.plugin.wrap_plugin
00362     def run(self, firstrun):
        """Schedule build or check for completion (and possibly install)"""
        if not os.path.exists(os.path.join(falcon.conf.rootdir, '.falcon', 'build', 'building', self.dscfile)):
            falcon.util.warning(_("File %s vanished. Will no longer attempt to build %s %s") % (self.dscfile, self.packagename, self.version))
            self.delete()
            return
        for b in self.active_builders.all():
            if firstrun:
                ret = b.build(self)
            else:
                ret = b.download(self)
            if ret == BUILD_PENDING:
                if firstrun:
                    falcon.util.output(_("Uploaded %s %s for building on %s buildd %s") % (self.packagename, self.version, b.arch, b.name))
                else:
                    falcon.util.output(_("Build %s %s is still pending on %s buildd %s") % (self.packagename, self.version, b.arch, b.name))
            elif ret == BUILD_FAILED:
                falcon.util.warning(_("Failed building %s %s on %s buildd %s") % (self.packagename, self.version, b.arch, b.name))
                self.active_builders.remove(b)
                self.inactive_builders.add(b)
                self.status[b.id] = (BUILD_FAILED, datetime.datetime.now())
            elif ret == BUILD_OK:
                falcon.util.output(_("Succeeded building %s %s on %s buildd %s") % (self.packagename, self.version, b.arch, b.name))
                self.active_builders.remove(b)
                self.inactive_builders.add(b)
                self.status[b.id] = (BUILD_OK, datetime.datetime.now())
            else:
                falcon.util.error(_("Unknown build result '%s'") % str(ret))

        # Final administration
        s = _("Build result for %s %s") % (self.packagename, self.version)
        falcon.util.output(s)
        falcon.util.output('=' * len(s))
        for b in self.active_builders.all():
            falcon.util.output("%-7s %-20s PENDING" % (b.arch, b.name))
        for b in self.inactive_builders.all():
            if self.status[b.id][0] == BUILD_FAILED:
                falcon.util.output("%-7s %-20s FAILED" % (b.arch, b.name))
            elif self.status[b.id][0] == BUILD_OK:
                falcon.util.output("%-7s %-20s OK" % (b.arch, b.name))
        self.save()

        if len(self.active_builders.all()):
            falcon.util.output(_("Run falcon-build-queue -c to check for finished builds"))
        else:
            falcon.util.output(_("Completely built %s %s") % (self.packagename, self.version))
            if self.install:
                if BUILD_FAILED in [x[0] for x in self.status.values()]:
                    falcon.util.warning(_("Can't install %s %s, building failed") % (self.packagename, version))
                else:
                    try:
                        cp = falcon.pocket.Component.objects.get(name = self.component, pocket__name = self.pocket)
                    except falcon.pocket.Component.DoesNotExist:
                        falcon.util.warning(_("Component %s/%s does not exist anymore, can't install %s %s") % 
                                            (self.pocket, self.component, self.packagename, self.version))
                        return
                    # Move the files
                    srcpath = os.path.join(falcon.conf.rootdir, '.falcon', 'build', 'building')
                    dsc = os.path.join(srcpath, self.dscfile)

                    control, controlfields, files = falcon.package.parse_dsc(dsc)
                    to_move = []
                    name = controlfields['Source']
                    for f in files:
                        if os.path.exists(os.path.join(srcpath,f.name)):
                            to_move.append(f.name)
                
                    binaries = [x.strip() for x in controlfields['Binary'].split(",")]
                    version = controlfields['Version']
                    if ':' in version:
                        version = version[version.find(':')+1:]
                    for b in binaries:
                        f = '%s_%s_all.deb' % (b, version)
                        if os.path.exists(os.path.join(srcpath,f)):
                            to_move.append(f)
                            continue
                        f = '%s_%s_all.udeb' % (b, version)
                        if os.path.exists(os.path.join(srcpath,f)):
                            to_move.append(f)
                            continue
                        for a in falcon.conf.architectures:
                            f = '%s_%s_%s.deb' % (b, version, a)
                            if os.path.exists(os.path.join(srcpath,f)):
                                to_move.append(f)
                                continue
                            f = '%s_%s_%s.udeb' % (b, version, a)
                            if os.path.exists(os.path.join(srcpath,f)):
                                to_move.append(f)
                                continue
                            # Tried'em all!
                            if falcon.conf.complete_only:
                                error(_("Can't install package %s because it was not yet completely built") % name)
                
                    # Find a changelog
                    cl = falcon.package.find_changelog(srcpath, to_move)
                    if cl:
                        to_move.append(os.path.basename(cl))
                
                    # Now install the package
                    for f in to_move:
                        if os.path.exists(os.path.join(cp.poolpath, f)):
                            if f.endswith('orig.tar.gz'):
                                to_move.remove(f)
                            else:
                                error(_("File %s already exists in component %s") % (f, cp.poolpath))
                    for f in to_move:
                        shutil.move(os.path.join(srcpath, f), os.path.join(cp.poolpath, f))
                        
                    s = falcon.package.SourcePackage.create_from_dscfile(cp, self.dscfile)
                    cp.install(s)
                    cp.save()
            # Always delete self when finished building
            self.delete()

Generated by  Doxygen 1.6.0   Back to index