Logo Search packages:      
Sourcecode: falcon version File versions  Download package

pocket.py

# This file is part of the Falcon repository manager
# Copyright (C) 2005-2008 Dennis Kaarsemaker
# See the file named COPYING in the root of the source tree for license details
#
# pocket.py - Functionality for maintaining pockets and components of a repository

import falcon
import apt_pkg; apt_pkg.init()
import os, shutil, time, UserDict
import cPickle as pickle
from gettext import gettext as _
from django.db import models
from django.template import Template, Context

pocket_release_template = Template("""Origin: {{ conf.origin }}
Label: {{ conf.origin }}
Suite: {{ conf.origin }}
Version: {{ p.version }}
Codename: {{ p.name }}
Architectures: {{ conf.architectures|join:" " }}
Components:{% for c in p.components %} {{ c.name }}{% endfor %}
Description: {{ conf.description }} version {{ p.version }}
MD5Sum:
{{ md5sums }}
SHA1:
{{ sha1sums }}
SHA256:
{{ sha256sums }}
""")
component_release_template = Template("""Archive: {{ c.pocket.name }}
Version: {{ c.pocket.version }}
Component: {{ c.name }}
Origin: {{ conf.origin }}
Label: {{ conf.origin }}
Architecture: {{ arch }}
""")

00038 class FileList(UserDict.DictMixin):
    """Dict-like object that can simply export to apt-file format"""
    def __init__(self):
        self.data = {}
    def __getitem__(self, key):
        return self.data[key]
    def __setitem__(self, key, val):
        if key in self.data:
            self.data[key].append(val)
        else:
            self.data[key] = [val]
    def __contains__(self, key):
        return key in self.data
    def __delitem__(self, key):
        del self.data[key]
    def keys(self):
        return self.data.keys()
    def __iter__(self):
        return self.data.iterkeys()
    def iteritems(self):
        return self.data.iteritems()

00060     def as_list(self):
        """Export to apt-file format"""
        return "\n".join(["%-60s    %s" % (x, ','.join(self.data[x])) for x in sorted(self.data.keys())])

00064 class Pocket(models.Model):
    """Abstraction of a repository pocket"""
    name = models.CharField(maxlength=30,unique=True)
    version = models.CharField(maxlength=10,default="0.0")
    description = models.TextField(default=_("No description"))

    def __init__(self, *args, **kwargs):
        super(Pocket, self).__init__(*args, **kwargs)
        self.poolpath = os.path.join('pool', self.name)
        self.distpath = os.path.join('dists', self.name)
        self._components = []

00076     def load_components(self):
        """Initialize new componenents"""
        existing = [x.name for x in self.realcomponents.all()]
        for d in os.listdir(self.poolpath):
            if os.path.isdir(os.path.join(self.poolpath, d)) and d not in existing:
                Component(pocket=self, name=d).save()

    def get_components(self):
        if not self._components:
            self._components = list(self.realcomponents.all()) + list(self.metacomponents.all())
            self._components.sort(lambda x,y: cmp(x.name,y.name))
        return self._components
    components = property(get_components)

    @falcon.plugin.wrap_plugin
00091     def export(self):
        """Export repository info to the metadata (Sources/Packages/Release)"""
        releasefiles = {}
        for c in self.components:
            releasefiles.update(c.releasefiles)

        md5sums = "".join([" %s %16d %s\n" % (releasefiles[name][1], releasefiles[name][0], name) for name in sorted(releasefiles.keys())])
        sha1sums = "".join([" %s %16d %s\n" % (releasefiles[name][2], releasefiles[name][0], name) for name in sorted(releasefiles.keys())])
        sha256sums = "".join([" %s %16d %s\n" % (releasefiles[name][3], releasefiles[name][0], name) for name in sorted(releasefiles.keys())])

        rel = pocket_release_template.render(Context({'conf': falcon.conf,
                                              'p': self,
                                              'md5sums': md5sums,
                                              'sha1sums': sha1sums,
                                              'sha256sums': sha256sums}))
        rf = os.path.join(self.distpath, 'Release')
        if falcon.util.writefile(rf, rel) or (falcon.conf.gpgkey and not os.path.exists(rf + '.gpg')):
            falcon.util.debug(_("Signing %s") % rf)
            if falcon.conf.gpgkey:
                if os.path.exists(rf + '.gpg'):
                    os.unlink(rf + '.gpg')
                try:
                    falcon.util.run(['gpg', '-u', falcon.conf.gpgkey, '--detach-sig', '--armor', '--output', '%s.gpg' % rf, rf])
                except:
                    falcon.util.warning(_("Failed to sign the release file"))

            # Generate contents list
            if falcon.conf.create_filelist:
                fl = {}
                for a in falcon.conf.architectures:
                    fl[a] = FileList()
                falcon.util.output(_("Creating file listings for pocket '%s'") % self.name)
                binaries = falcon.package.BinaryPackage.objects.filter(sourcepackage__component__pocket = self)
                for b in binaries:
                    tag = '%s/%s' % (b.controlfields['Section'], b.packagename)
                    for f in b.files:
                        if not f.endswith('/'):
                            if b.architecture == 'all':
                                for a in falcon.conf.architectures:
                                    fl[a][f] = tag
                            else:
                                fl[b.architecture][f] = tag
                for a in falcon.conf.architectures:
                    fo = os.path.join(self.distpath, 'Contents-%s' % a)
                    falcon.util.writefile(fo, fl[a].as_list())
                    falcon.util.run(['gzip', '-f', fo])

        if falcon.conf.webbase:
            appinstall = False
            try:
                appinstall = falcon.plugins.app_install.AppInstallDataPlugin.conf.enabled
            except:
                pass
            template = falcon.util.get_template('pocket.html')
            context = Context({'p': self, 'conf': falcon.conf, 'dots': '../../','components': self.components, 'appinstall': appinstall})
            falcon.util.writefile(os.path.join(self.distpath, 'index.html'), template.render(context))

    def __str__(self):
        return self.name

    def __repr__(self):
        return self.name

00154 class Component(models.Model):
    """Abstraction of a repository component"""
    pocket       = models.ForeignKey(Pocket, related_name='realcomponents')
    name         = models.CharField(maxlength=50)
    keepfiles    = models.PickleField(default = [])
    morguefiles  = models.PickleField(default = {})
    releasefiles = models.PickleField(default = {})
    description  = models.TextField(default=_("No description"))

    def __init__(self, *args, **kwargs):
        super(Component, self).__init__(*args, **kwargs)
        self.path     = os.path.join(self.pocket.name, self.name)
        self.poolpath = os.path.join('pool', self.path)
        self.distpath = os.path.join('dists', self.path)
        if type(self.keepfiles) == str:
            self.keepfiles = pickle.loads(self.keepfiles)
        if type(self.morguefiles) == str:
            self.morguefiles = pickle.loads(self.morguefiles)
        if type(self.releasefiles) == str:
            self.releasefiles = pickle.loads(self.releasefiles)

    @falcon.plugin.wrap_plugin
00176     def scan(self, clean=False):
        """Scan for new/updated packages and extract metadata"""
        if clean:
            self.keepfiles = []
            self.save()

        files = os.listdir(self.poolpath)
        for s in self.sources.all():
            newfiles = s.rescan()
            for f in newfiles:
                # Guard against orig.tar.gz being removed twice
                if f in files:
                    files.remove(f)

        sources = [x for x in files if x.endswith('.dsc') and x not in self.morguefiles]
        for s in sources:
            s = falcon.package.SourcePackage.create_from_dscfile(self, s)
            if falcon.conf.complete_only and not s.is_complete():
                falcon.util.warning(_("Can't install incomplete sourcepackage %s %s") % s.package)
            else:
                p = self.install(s)
            for f in s.files:
                # Guard against orig.tar.gz being removed twice
                if f in files:
                    files.remove(f)

        # Create pseudo sources for sourceless debs
        debs = [f for f in files if f.endswith('.deb') or f.endswith('.udeb')]
        fakesources = []
        for d in debs:
            files.remove(d)
            s = falcon.package.SourcePackage.create_fake_source(self, d)
            if s:
                self.install(s)

        # Remove non-package files
        for f in sorted(files):
            if f in self.keepfiles:
                continue
            if f.endswith('.changes'):
                os.rename(os.path.join(self.poolpath, f), os.path.join('.falcon', 'changelogs', f))
            elif falcon.questions.yesno(_("Remove non-package file %s?") % f, False):
                morgue(os.path.join(self.poolpath, f))
            else:
                self.keepfiles.append(f)

        # Morgue packages
        for m in self.morguefiles:
            key = "pkg:%s_%s" % m
            if key in self.keepfiles:
                continue
            if falcon.questions.yesno(_("Move old version %s of package %s to the morgue?") % (m[1], m[0])):
                pkgs = self.sources.filter(packagename=m[0]).exclude(version=m[1])
                for f in self.morguefiles[m]:
                    # Don't remove files needed by newer versions
                    for p in pkgs:
                        if f in p.files:
                           break
                    else:
                        morgue(os.path.join(self.poolpath, f))
            else:
                self.keepfiles.append(key)
        self.morguefiles = {}
        self.save()

    @falcon.plugin.wrap_plugin
00242     def install(self, package):
        """Install a sourcepackage into a component. Returns the installed package if succesful, or False
           if a newer version of the package has been installed before. Will morgue older versions of the
           package that have been installed earlier"""
        try:
            oldpackage = self.sources.extra(where=["id != %s"], params = [package.id]).get(component=self, packagename=package.packagename)
        except falcon.package.SourcePackage.DoesNotExist:
            return package
        
        vc = apt_pkg.VersionCompare(package.version, oldpackage.version)
        if vc == 0:
            package.delete()
            oldpackage.delete()
            falcon.util.error(_("You have two sources with the same name and version, can't do that!"))
        elif vc == -1:
            self.morguefiles[(package.packagename, package.version)] = package.files
            package.delete()
            return False
        else:
            self.morguefiles[(oldpackage.packagename, oldpackage.version)] = oldpackage.files
            oldpackage.delete()
            falcon.conf.lastinstall = time.time()
            return package

    @falcon.plugin.wrap_plugin
00267     def export(self):
        """Create Packages/Sources lists"""
        sources = ''
        binaries = dict(zip(falcon.conf.architectures, ['']*len(falcon.conf.architectures)))
        for s in self.sources.all():
            if not s.is_fake:
                sources += s.control + "\n\n"
            for p in s.binaries.all():
                if p.architecture == 'all':
                    for arch in falcon.conf.architectures:
                        binaries[arch] += p.control + "\n\n"
                else:
                    binaries[p.architecture] += p.control + "\n\n"

        # Write exports
        self.releasefiles.clear()
        sf = os.path.join(self.distpath, 'source', 'Sources')
        sf2 = '%s/source/Sources' % self.name
        if falcon.util.writefile(sf, sources):
            falcon.util.run(['gzip', '-c', sf], outfile='%s.gz' % sf)
            falcon.util.run(['bzip2', '-k', '-f', sf])

        self.releasefiles[sf2] = (len(sources), apt_pkg.md5sum(sources), apt_pkg.sha1sum(sources), apt_pkg.sha256sum(sources))
        fd = open(sf + '.gz')
        md5 = apt_pkg.md5sum(fd); fd.seek(0)
        sha1 = apt_pkg.sha1sum(fd); fd.seek(0)
        sha256 = apt_pkg.sha256sum(fd); fd.close()
        self.releasefiles[sf2+'.gz'] = (os.path.getsize(sf+'.gz'), md5, sha1, sha256)
        fd = open(sf + '.bz2')
        md5 = apt_pkg.md5sum(fd); fd.seek(0)
        sha1 = apt_pkg.sha1sum(fd); fd.seek(0)
        sha256 = apt_pkg.sha256sum(fd); fd.close()
        self.releasefiles[sf2+'.bz2'] = (os.path.getsize(sf+'.bz2'), md5, sha1, sha256)

        rf = os.path.join(self.distpath, 'source', 'Release')
        rf2 = '%s/source/Release' % self.name
        rel = component_release_template.render(Context({'c': self, 'arch': 'source', 'conf': falcon.conf}))
        falcon.util.writefile(rf, rel)
        self.releasefiles[rf2] = (len(rel), apt_pkg.md5sum(rel), apt_pkg.sha1sum(rel), apt_pkg.sha256sum(rel))

        for a in binaries:
            bf = os.path.join(self.distpath, 'binary-%s' % a, 'Packages')
            bf2 = '%s/binary-%s/Packages' % (self.name, a)
            if falcon.util.writefile(bf, binaries[a]):
                falcon.util.run(['gzip', '-c', bf], outfile='%s.gz' % bf)
                falcon.util.run(['bzip2', '-k', '-f', bf])

            self.releasefiles[bf2] = (len(binaries[a]), apt_pkg.md5sum(binaries[a]), apt_pkg.sha1sum(binaries[a]), apt_pkg.sha256sum(binaries[a]))
            fd = open(bf + '.gz')
            md5 = apt_pkg.md5sum(fd); fd.seek(0)
            sha1 = apt_pkg.sha1sum(fd); fd.seek(0)
            sha256 = apt_pkg.sha256sum(fd); fd.close()
            self.releasefiles[bf2+'.gz'] = (os.path.getsize(bf+'.gz'), md5, sha1, sha256)
            fd = open(bf + '.bz2')
            md5 = apt_pkg.md5sum(fd); fd.seek(0)
            sha1 = apt_pkg.sha1sum(fd); fd.seek(0)
            sha256 = apt_pkg.sha256sum(fd); fd.close()
            self.releasefiles[bf2+'.bz2'] = (os.path.getsize(bf+'.bz2'), md5, sha1, sha256)

            rf = os.path.join(self.distpath, 'binary-%s' % a, 'Release')
            rf2 = '%s/binary-%s/Release' % (self.name, a)
            rel = component_release_template.render({'c': self, 'arch': a, 'conf': falcon.conf})
            falcon.util.writefile(rf, rel)
            self.releasefiles[rf2] = (len(rel), apt_pkg.md5sum(rel), apt_pkg.sha1sum(rel), apt_pkg.sha256sum(rel))

        self.save()

        if falcon.conf.webbase:
            template = falcon.util.get_template('component.html')
            context = Context({'c': self, 'conf': falcon.conf, 'dots': '../../../', 
                               'sources': sorted(self.sources.all(), lambda x,y: cmp(x.packagename, y.packagename))})
            falcon.util.writefile(os.path.join(self.distpath, 'index.html'), template.render(context))

    def __str__(self):
        return self.path

00343 class MetaComponent(models.Model):
    """Abstraction of a metacomponent"""
    pocket       = models.ForeignKey(Pocket, related_name='metacomponents')
    name         = models.CharField(maxlength=50)
    releasefiles = models.PickleField(default = {})
    description  = models.TextField(default=_("No description"))
    components   = models.ManyToManyField(Component)

    def __init__(self, *args, **kwargs):
        super(MetaComponent, self).__init__(*args, **kwargs)
        self.path     = os.path.join(self.pocket.name, self.name)
        self.distpath = os.path.join('dists', self.path)

        if type(self.releasefiles) == str:
            self.releasefiles = pickle.loads(self.releasefiles)
   
    def scan(self): return
    def clean(self): return
    def install(self, package):
        falcon.util.error(_("Can't install into a metacomponent"))

    @falcon.plugin.wrap_plugin
00365     def export(self):
        """Create Packages/Sources lists"""
        self.sources = []
        for c in self.components.all():
            self.sources += c.sources.all()

        # Copied from Component
        sources = ''
        binaries = dict(zip(falcon.conf.architectures, ['']*len(falcon.conf.architectures)))
        for s in self.sources:
            if not s.is_fake:
                sources += s.control + "\n\n"
            for p in s.binaries.all():
                if p.architecture == 'all':
                    for arch in falcon.conf.architectures:
                        binaries[arch] += p.control + "\n\n"
                else:
                    binaries[p.architecture] += p.control + "\n\n"

        # Write exports
        self.releasefiles.clear()
        sf = os.path.join(self.distpath, 'source', 'Sources')
        sf2 = '%s/source/Sources' % self.name
        if falcon.util.writefile(sf, sources):
            falcon.util.run(['gzip', '-c', sf], outfile='%s.gz' % sf)
            falcon.util.run(['bzip2', '-k','-f', sf])

        self.releasefiles[sf2] = (len(sources), apt_pkg.md5sum(sources), apt_pkg.sha1sum(sources), apt_pkg.sha256sum(sources))
        fd = open(sf + '.gz')
        md5 = apt_pkg.md5sum(fd); fd.seek(0)
        sha1 = apt_pkg.sha1sum(fd); fd.seek(0)
        sha256 = apt_pkg.sha256sum(fd); fd.close()
        self.releasefiles[sf2+'.gz'] = (os.path.getsize(sf+'.gz'), md5, sha1, sha256)
        fd = open(sf + '.bz2')
        md5 = apt_pkg.md5sum(fd); fd.seek(0)
        sha1 = apt_pkg.sha1sum(fd); fd.seek(0)
        sha256 = apt_pkg.sha256sum(fd); fd.close()
        self.releasefiles[sf2+'.bz2'] = (os.path.getsize(sf+'.bz2'), md5, sha1, sha256)

        rf = os.path.join(self.distpath, 'source', 'Release')
        rf2 = '%s/source/Release' % self.name
        rel = component_release_template.render(Context({'c': self, 'arch': 'source', 'conf': falcon.conf}))
        falcon.util.writefile(rf, rel)
        self.releasefiles[rf2] = (len(rel), apt_pkg.md5sum(rel), apt_pkg.sha1sum(rel), apt_pkg.sha256sum(rel))

        for a in binaries:
            bf = os.path.join(self.distpath, 'binary-%s' % a, 'Packages')
            bf2 = '%s/binary-%s/Packages' % (self.name, a)
            if falcon.util.writefile(bf, binaries[a]):
                falcon.util.run(['gzip', '-c', bf], outfile='%s.gz' % bf)
                falcon.util.run(['bzip2', '-k', '-f', bf])

            self.releasefiles[bf2] = (len(binaries[a]), apt_pkg.md5sum(binaries[a]), apt_pkg.sha1sum(binaries[a]), apt_pkg.sha256sum(binaries[a]))
            fd = open(bf + '.gz')
            md5 = apt_pkg.md5sum(fd); fd.seek(0)
            sha1 = apt_pkg.sha1sum(fd); fd.seek(0)
            sha256 = apt_pkg.sha256sum(fd); fd.close()
            self.releasefiles[bf2+'.gz'] = (os.path.getsize(bf+'.gz'), md5, sha1, sha256)
            fd = open(bf + '.bz2')
            md5 = apt_pkg.md5sum(fd); fd.seek(0)
            sha1 = apt_pkg.sha1sum(fd); fd.seek(0)
            sha256 = apt_pkg.sha256sum(fd); fd.close()
            self.releasefiles[bf2+'.bz2'] = (os.path.getsize(bf+'.bz2'), md5, sha1, sha256)

            rf = os.path.join(self.distpath, 'binary-%s' % a, 'Release')
            rf2 = '%s/binary-%s/Release' % (self.name, a)
            rel = component_release_template.render({'c': self, 'arch': a, 'conf': falcon.conf})
            falcon.util.writefile(rf, rel)
            self.releasefiles[rf2] = (len(rel), apt_pkg.md5sum(rel), apt_pkg.sha1sum(rel), apt_pkg.sha256sum(rel))

        self.save()

        if falcon.conf.webbase:
            template = falcon.util.get_template('component.html')
            context = Context({'c': self, 'conf': falcon.conf, 'dots': '../../../', 
                               'sources': sorted(self.sources, lambda x,y: cmp(x.packagename, y.packagename))})
            falcon.util.writefile(os.path.join(self.distpath, 'index.html'), template.render(context))

    def __str__(self):
        return self.path

@falcon.plugin.wrap_plugin
def morgue(file):
    """Move a file to the morgue, or delete it if it already exists in the morgue"""
    fn = os.path.basename(file)
    if os.path.exists(os.path.join('.falcon', 'morgue', fn)):
        if falcon.questions.yesno(_("File %s already exists in the morgue, delete non-morgued version?") % fn, False):
            if(os.path.isdir(file)):
                shutil.rmtree(file)
            else:
                os.unlink(file)
        else:
            return False
    else:
        os.rename(file, os.path.join('.falcon', 'morgue', fn))
    return True

Generated by  Doxygen 1.6.0   Back to index