Logo Search packages:      
Sourcecode: falcon version File versions  Download package

package.py

# This file is part of the Falcon repository manager
# Copyright (C) 2005-2008 Dennis Kaarsemaker
# See the file named COPYING in the root of the source tree for license details
#
# package.py - Functionality for handling source- and binary packages

import falcon
from email import FeedParser
import os, re, apt_pkg, apt_inst, md5, chardet
from django.db import models
import cPickle as pickle

00013 class SourcePackage(models.Model):
    """Abstraction of a source package"""
    component = models.ForeignKey('Component', related_name='sources')
    is_fake = models.BooleanField(default=False)
    filename = models.CharField(maxlength=70)
    mtime = models.IntegerField()
    packagename = models.CharField(maxlength=50)
    version = models.CharField(maxlength=15)
    control = models.TextField()
    controlfields = models.PickleField(default={})
    files = models.PickleField(default=[])
    sourcefiles = models.PickleField(default = [])
    changelog = models.TextField()

    def __init__(self, *args, **kwargs):
        super(SourcePackage, self).__init__(*args, **kwargs)
        if type(self.controlfields) == str:
            self.controlfields = pickle.loads(self.controlfields)
        if type(self.files) == str:
            self.files = pickle.loads(self.files)
        if type(self.sourcefiles) == str:
            self.sourcefiles = pickle.loads(self.sourcefiles)

    @classmethod
00037     def create_from_dscfile(cls, component, filename):
        """Create a SourcePackage instance, given a component and the name of a
           .dsc file inside that component"""
        control, controlfields, files = parse_dsc(os.path.join(component.poolpath, filename))
        c =  cls(component = component,
                 is_fake = False,
                 filename = filename,
                 mtime = os.path.getmtime(os.path.join(component.poolpath, filename)),
                 packagename = controlfields['source'],
                 version = controlfields['version'],
                 control = control,
                 controlfields = controlfields,
                 files = [x.name for x in files],
                 changelog = '',
                 sourcefiles = files)
        for f in c.files[:]:
            if not os.path.exists(os.path.join(c.component.poolpath, f)):
                falcon.util.warning(_("File %s seems to be missing!") % f)
                c.files.remove(f)
        # Extra save to fill in the id 
        c.save()
        c.get_binaries()
        c.changelog = get_changelog(c.component.poolpath, c.files)
        c.save()
        return c

    @classmethod
00064     def create_fake_source(cls, component, debfile):
        """Create a SourcePackage instance, given a component and the name of a
           .deb file inside that component"""
        control, controlfields, files = parse_deb(debfile, component)
        # Some packages really don't have a Source: header (make-jpkg builds those for instance)
        if not controlfields['Source']:
            controlfields['Source'] = controlfields['Package']
        # Remove +.... from the sourcepackage, module-assistant adds the kernel version this way
        source_version = controlfields['Version']
        if '+' in source_version:
            source_version = source_version[:source_version.find('+')]
        ret_s = None
        try:
            s = cls.objects.get(component = component, packagename = controlfields['Source'], version = source_version)
            if not s.is_fake:
                falcon.util.error(_('Non-source deb %s specifies %s as source, but source %s does not specify %s as binary') %
                                 (debfile, controlfields['Source'], s.packagename, controlfields['Package']))
        except cls.DoesNotExist:
            s = cls(component=component,
                    is_fake=True,
                    filename='',
                    mtime=0,
                    packagename = controlfields['source'],
                    version = source_version,
                    control = '',
                    controlfields = {'Directory': component.poolpath, 'directory': component.poolpath,
                                     'Maintainer': controlfields['Manitainer'], 'maintainer': controlfields['maintainer']},
                    files = [],
                    changelog = '',
                    sourcefiles = []
                   )
            ret_s = s
            s.save()

        # Does binary exist? If so: bail out!
        for b in s.binaries.all():
            if b.architecture == controlfields['Architecture'] and b.packagename == controlfields['Package']:
                falcon.util.error(_('Duplicate binary package %s detected') % debfile)

        # Add binary package
        b = BinaryPackage(sourcepackage=s,
                          filename = debfile,
                          mtime = os.path.getmtime(os.path.join(component.poolpath, debfile)),
                          packagename = controlfields['Package'],
                          architecture = controlfields['Architecture'],
                          version = controlfields['Version'],
                          control = control,
                          controlfields = controlfields,
                          files = files
                         )
        b.save()
        s.files.append(debfile)
        s.save()
        return ret_s
                   
00119     def get_binaries(self, missing_only=False):
        """Find the binary packages that belong to this SourcePackage"""

        # Strip epoch
        eversion = version = self.version
        if ':' in version:
            eversion = version[version.find(':')+1:]

        binaries = [x.strip() for x in self.controlfields['Binary'].split(",")]

        for b in binaries:
            try:
                BinaryPackage.objects.get(packagename=b, architecture='all', version=version, sourcepackage__component=self.component)
                continue
            except BinaryPackage.DoesNotExist:
                pass
            f = '%s_%s_all.deb' % (b, eversion)
            if os.path.exists(os.path.join(self.component.poolpath, f)):
                BinaryPackage.create_from_debfile(self, f)
                self.files.append(f)
                continue
            f = '%s_%s_all.udeb' % (b, eversion)
            if os.path.exists(os.path.join(self.component.poolpath, f)):
                BinaryPackage.create_from_debfile(self, f)
                self.files.append(f)
                continue

            # Find arch specific packages
            for a in falcon.conf.architectures:
                try:
                    BinaryPackage.objects.get(packagename=b, architecture=a, version=version, sourcepackage__component=self.component)
                    continue
                except BinaryPackage.DoesNotExist:
                    pass
                f = '%s_%s_%s.deb' % (b, eversion, a)
                if os.path.exists(os.path.join(self.component.poolpath, f)):
                    BinaryPackage.create_from_debfile(self, f)
                    self.files.append(f)
                    continue
                f = '%s_%s_%s.udeb' % (b, eversion, a)
                if os.path.exists(os.path.join(self.component.poolpath, f)):
                    BinaryPackage.create_from_debfile(self, f)
                    self.files.append(f)

            for a in falcon.conf.architectures:
                try:
                    BinaryPackage.objects.get(packagename=b + '-dbgsym', architecture=a, version=version, sourcepackage__component=self.component)
                    continue
                except BinaryPackage.DoesNotExist:
                    pass
                f = '%s-dbgsym_%s_%s.ddeb' % (b, eversion, a)
                if os.path.exists(os.path.join(self.component.poolpath, f)):
                    BinaryPackage.create_from_debfile(self, f)
                    self.files.append(f)

00174     def rescan(self):
        """Scan the component for existence/updates of this package and its
           binary packages"""
        if not self.is_fake and not os.path.exists(os.path.join(self.component.poolpath, self.filename)):
            self.delete() # Also deletes its binaries
            return []

        if not self.is_fake and (os.path.getmtime(os.path.join(self.component.poolpath, self.filename)) != self.mtime):
            falcon.util.warning("Source package %s changed after installing" % self.filename)
            self.control, self.controlfields, files = parse_dsc(os.path.join(self.component.poolpath, self.filename))
            self.files = [x.name for x in files]
            self.mtime = os.path.getmtime(os.path.join(self.component.poolpath, self.filename))
            self.packagename = self.controlfields['source']
            self.version = self.controlfields['version']
            self.changelog = get_changelog(self.component.poolpath, self.files)
            must_save = True
        for f in self.files[:]:
            if not os.path.exists(os.path.join(self.component.poolpath, f)):
                falcon.util.warning(_("File %s seems to be missing!") % f)
                self.files.remove(f)
        for b in self.binaries.all():
            if not b.rescan():
                if b.filename in self.files:
                    self.files.remove(b.filename)
            else:
                if b.filename not in self.files:
                    self.files.append(b.filename)

        if not self.is_fake:
            self.get_binaries(missing_only=True)
        self.save()
        return self.files

    def sorted_binaries(self):
        binaries = []
        seen = []
        for b in self.binaries.all():
            if b.packagename not in seen:
                seen.append(b.packagename)
                b.packages = [b.filename]
                binaries.append(b)
            else:
                for b2 in binaries:
                    if b2.packagename == b.packagename:
                        b2.packages.append(b.filename)
        return sorted(binaries, lambda x,y: cmp(x.packagename, y.packagename))
    
00221     def is_complete(self):
        """Check if all binary packages are built"""
        binaries = [x.strip() for x in self.controlfields['Binary'].split(",")]
        for b in binaries:
            try:
                BinaryPackage.objects.get(packagename=b, sourcepackage=self, architecture='all')
            except BinaryPackage.DoesNotExist:
                archs = self.controlfields.get('Architecture',None).strip()
                if not archs:
                    return True # Fake source, can't say anything about it
                if archs == 'any':
                    archs = falcon.conf.architectures
                else:
                    archs = archs.split(' ,')
                for a in archs:
                    try:
                        BinaryPackage.objects.get(packagename=b, sourcepackage=self, architecture=a)
                    except BinaryPackage.DoesNotExist:
                        return False
        return True

    def __repr__(self):
        if self.is_fake:
            return '<Sourcepackage %s (fake)>' % (self.packagename)
        return '<Sourcepackage %s (%s)>' % (self.packagename, self.filename)
        
    def __str__(self):
        if self.is_fake:
            return 'Sourcepackage %s (fake)' % (self.packagename)
        return 'Sourcepackage %s from %s' % (self.packagename, self.filename)

00252 class BinaryPackage(models.Model):
    """Abstraction of a binary package"""
    sourcepackage = models.ForeignKey(SourcePackage, related_name='binaries')
    filename = models.CharField(maxlength=70)
    mtime = models.IntegerField()
    packagename = models.CharField(maxlength=50)
    architecture = models.CharField(maxlength=10)
    version = models.CharField(maxlength=15)
    control = models.TextField()
    controlfields = models.PickleField(default={})
    files = models.PickleField(default=[])
    app_install_files = models.PickleField(default={})

    def __init__(self, *args, **kwargs):
        super(BinaryPackage, self).__init__(*args, **kwargs)
        if type(self.controlfields) == str:
            self.controlfields = pickle.loads(self.controlfields)
        if type(self.files) == str:
            self.files = pickle.loads(self.files)
        if type(self.app_install_files) == str:
            self.app_install_files = pickle.loads(self.app_install_files)

    @classmethod
00275     def create_from_debfile(cls, source, filename):
        """Create a BinaryPackage instance, given a SourcePackage and the name
           of a .deb file belonging to that package"""
        control, controlfields, files = parse_deb(filename, source.component)
        p = cls(
            sourcepackage = source,
            filename = filename,
            mtime = os.path.getmtime(os.path.join(source.component.poolpath, filename)),
            packagename = controlfields['Package'],
            architecture = controlfields['Architecture'],
            version = controlfields['Version'],
            control = control,
            controlfields = controlfields,
            files = files
        )
        p.save()

00292     def rescan(self):
        """Scan the component for existence/updates of this package"""
        if not os.path.exists(os.path.join(self.sourcepackage.component.poolpath, self.filename)):
            self.delete()
            return False
        if os.path.getmtime(os.path.join(self.sourcepackage.component.poolpath, self.filename)) != self.mtime:
            falcon.util.warning("Binary package %s changed after installing" % self.filename)
            self.control, self.controlfields, self.files = parse_deb(self.filename, self.sourcepackage.component)
            self.mtime = os.path.getmtime(os.path.join(self.sourcepackage.component.poolpath, self.filename))
            self.packagename = self.controlfields['Package']
            self.architecture = self.controlfields['Architecture']
            self.version = self.controlfields['Version']
            self.save()
        return True

    def __repr__(self):
        return '<Binary package %s (%s)>' % (self.packagename, self.filename)

    def __str__(self):
        return 'Binary package %s from %s' % (self.packagename, self.filename)

00313 class file(object):
    """Small object representing a Files: line in a .dsc/.changes file"""
    def __init__(self, sum, size, name):
        self.name = name
        self.size = int(size)
        self.sum = sum
        for _type in ('.dsc', 'orig.tar.gz', '.diff.gz', '.tar.gz'):
            if name.endswith(_type):
                self.type = _type
                break

# Low level parsing functions
signed_re = re.compile(r'^----.*?\n\n(.*?)\n\n.*$', re.DOTALL)
files_re = re.compile(r'(\nFiles:\s*\n)')
def parse_dsc(filename):
    """Parse a possibly signed .dsc file into a dictionary"""
    dsc = falcon.util.readfile(filename)
    sum = md5.new(dsc).hexdigest()
    dsc = unicode(dsc.strip(), chardet.detect(dsc.strip())['encoding']).encode('utf-8')
    path = os.path.dirname(filename)
    size = os.path.getsize(filename)
    # Insert Directory and extra file
    if dsc.startswith('---'):
        dsc = signed_re.sub(r'\1', dsc)
    dsc = files_re.sub('\nDirectory: %s\\1 %s %s %s\n' % (path, sum, size, os.path.basename(filename)), dsc)
    parser = FeedParser.FeedParser()
    parser.feed(dsc)
    controlfields = parser.close()
    files = []
    for f in controlfields['Files'].split('\n'):
        f = f.strip()
        if not f:
            continue
        try:
            sum, size, name = f.split()
        except ValueError: # Cheat, make it suitable to parse _arch.changes
            sum, size, section, prority, name = f.split()
        files.append(file(sum, size, name))
    return dsc, controlfields, files

section_re = re.compile(r'(\nSection:\s*)')
def parse_deb(filename, component=None):
    """Disect a .deb file and extract the needed info"""
    if component:
        filename = os.path.join(component.poolpath, filename)
    fd = open(filename)
    try:
        control = apt_inst.debExtractControl(fd).strip()
    except SystemError:
        falcon.util.error(_("%s seems to be an invalid .deb file") % filename)
    control = unicode(control, chardet.detect(control)['encoding']).encode('utf-8')
    if component and (component.name != 'main'):
        control = section_re.sub('\\1%s/' % component.name, control)
    fd.seek(0)
    md5sum = apt_pkg.md5sum(fd); fd.seek(0)
    sha1sum = apt_pkg.sha1sum(fd); fd.seek(0)
    sha256sum = apt_pkg.sha256sum(fd); fd.seek(0)
    control += "\nFilename: %s\nSize: %d" % (filename, os.path.getsize(filename))
    control += "\nMD5sum: %s\nSHA1: %s\nSHA256: %s\nOrigin: %s" % (md5sum, sha1sum, sha256sum, falcon.conf.origin)
    if falcon.conf.bugs and ('\nBugs' not in control):
        control += '\nBugs: %s' % falcon.conf.bugs
    parser = FeedParser.FeedParser()
    parser.feed(control)
    controlfields = parser.close()
    files = []

    if falcon.conf.create_filelist:
        # Seek to third member of the ar archive and use its filename
        # http://en.wikipedia.org/wiki/Ar_(Unix)
        pos = 8
        fd.seek(pos + 48)
        size = int(fd.read(10).strip())
        pos += 60 + size
        fd.seek(pos + 48)
        size = int(fd.read(10).strip())
        pos += 60 + size
        fd.seek(pos)
        fn = fd.read(16).strip()
        fd.seek(0)
        apt_inst.debExtract(fd, filecallback(files), fn)

    return control, controlfields, files

# Small helper for python_apt
def filecallback(lst):
    def cb(ign, filename, *args):
        lst.append(filename)
    return cb

def get_changelog(path, files):
    f = find_changelog(path, files)
    if f: 
        return falcon.util.readfile(f)
    return ''

def find_changelog(path, files):
    """Find a changelog for a set of files"""
    real_files = [os.path.join(path, f) for f in files]
    cached_files = [os.path.join('.falcon','changelogs',f) for f in files]
    for f in real_files + cached_files:
        if f.endswith('dsc'):
            f = f.replace('dsc','source.changes')
            if os.path.exists(f):
                return f
        elif f.endswith('all.deb'):
            for a in falcon.conf.architectures:
                f2 = f.replace('all.deb', '%s.changes' % a)
                if os.path.exists(f2):
                    return f2
        elif f.endswith('deb'):
            f = f.replace('deb','changes')
            if os.path.exists(f):
                return f

Generated by  Doxygen 1.6.0   Back to index