- update to be more generic

[projects/pld-ftp-admin.git] / modules / ftptree.py

# vi: encoding=utf-8 ts=8 sts=4 sw=4 et

import os, config, string, urllib, re, rpm
from common import fileexists, noarchcachedir
from baseftptree import BasePkg, BaseFtpTree
errnum=0

quietmode=False

class SomeError(Exception):
    def __init__(self):
        return

    def __str__(self):
        print "","An Error occured!"

def bailoutonerror():
    if not errnum == 0:
        print "%d error(s) encountered... aborting" % errnum
        raise SomeError

def pinfo(msg):
    print 'INFO: ' + msg

def perror(msg):
    global errnum
    errnum=errnum+1
    print 'ERR: ' + msg

def pwarning(msg):
    print 'WARN: ' + msg

def rm(file, test=False):
    if test:
        if not os.path.exists(file):
            pinfo("TEST os.remove(%s): file doesn't exists" % file)
    else:
        try:
            os.remove(file)
        except OSError, e:
            pinfo("os.remove(%s): %s" % (file, e))
            raise

def mv(src, dst, test=False):
    fsrc = src
    fdst = dst+'/'+src.split('/')[-1]
    if test:
        if not os.path.exists(src):
            pinfo("TEST os.rename(%s, %s): source doesn't exists" % (fsrc, fdst))
    else:
        try:
            os.rename(fsrc, fdst)
        except OSError, e:
            pinfo("os.rename(%s, %s): %s" % (fsrc, fdst, e))
            raise

class Pkg(BasePkg):
    def __init__(self, nvr, tree):
        BasePkg.__init__(self, nvr, tree)
        self.name=string.join(nvr.split('-')[:-2], '-')
        self.version=nvr.split('-')[-2]
        self.release=nvr.split('-')[-1]
        self.marked4removal=False
        self.marked4moving=False
        self.marked4movingpool=[]
        self.errors=[]
        self.warnings=[]

    def __cmp__(self, pkg):
        if self.name > pkg.name:
            return 1
        elif self.name < pkg.name:
            return -1
        else:
            return rpm.labelCompare(('0', self.version, self.release),
                                    ('0', pkg.version, pkg.release))

    def mark4moving(self):
        if not self.marked4moving:
            # Only one pkg in this pool can be marked for moving
            for pkg in self.marked4movingpool:
                pkg.unmark4moving()
            self.tree.marked4moving.append(self)
            self.marked4moving=True

    def unmark4moving(self):
        if self.marked4moving:
            self.tree.marked4moving.remove(self)
            self.marked4moving=False

    def mark4removal(self):
        if not self.marked4removal:
            self.tree.marked4removal.append(self)
            self.marked4removal=True

    def error(self, msg):
        self.errors.append(msg)
        if not quietmode:
            perror('%s %s' % (self.nvr, msg))

    def warning(self, msg):
        self.warnings.append(msg)
        if not quietmode:
            pwarning('%s %s' % (self.nvr, msg))

    def load(self, content=None):
        BasePkg.load(self, content)
        if self.info.has_key('move'):
            self.mark4moving()

    def writeinfo(self):
        f=open(self.tree.basedir+'/SRPMS/.metadata/'+self.nvr+'.src.rpm.info', 'w')
        for bid in self.build.keys():
            f.write("info:build:%s:requester:%s\ninfo:build:%s:requester_email:%s\n" % (bid, self.build[bid].requester, bid, self.build[bid].requester_email))
        for key in self.info.keys():
            f.write("info:%s:%s\n" % (key, string.join(self.info[key], ':')))
        for arch in self.files.keys():
            for rpm in self.files[arch]:
                f.write("file:%s:%s\n" % (arch, rpm))
        
    def remove(self, test=False):
        for arch in self.files.keys():
            for rpm in self.files[arch]:
                rm(self.tree.basedir+'/'+arch+'/RPMS/'+rpm, test)
                if arch=='noarch':
                    if fileexists(noarchcachedir+rpm+'.filelist'):
                        rm(noarchcachedir+rpm+'.filelist', test)
                    if fileexists(noarchcachedir+rpm+'.reqlist'):
                        rm(noarchcachedir+rpm+'.reqlist', test)
        rm(self.tree.basedir+'/SRPMS/.metadata/'+self.nvr+'.src.rpm.info', test)

    def move(self, dsttree, test=False):
        if dsttree.has_key(self.nvr):
            movedany=False
            for arch in self.files.keys():
                if arch in dsttree[self.nvr].files.keys():
                    msg = ""
                    if test:
                        msg = "TEST "
                    pinfo("%sArch %s for %s is already present in dest tree; removing from srctree" % (msg, arch, self.nvr))
                    for rpm in self.files[arch]:
                        rm(self.tree.basedir+'/'+arch+'/RPMS/'+rpm, test)
                else:
                    movedany=True
                    dsttree[self.nvr].files[arch]=self.files[arch]
                    for rpm in self.files[arch]:
                        mv(self.tree.basedir+'/'+arch+'/RPMS/'+rpm, dsttree.basedir+'/'+arch+'/RPMS/', test)
            if not test and  movedany:
                for bid in self.build.keys():
                    dsttree[self.nvr].build[bid]=self.build[bid]
                dsttree[self.nvr].writeinfo()
            rm(self.tree.basedir+'/SRPMS/.metadata/'+self.nvr+'.src.rpm.info', test)
        else:
            for arch in self.files.keys():
                for rpm in self.files[arch]:
                    mv(self.tree.basedir+'/'+arch+'/RPMS/'+rpm, dsttree.basedir+'/'+arch+'/RPMS/', test)
            mv(self.tree.basedir+'/SRPMS/.metadata/'+self.nvr+'.src.rpm.info', dsttree.basedir+'/SRPMS/.metadata/', test)


class FtpTree(BaseFtpTree):
    def __init__(self, tree, loadall=False):
        BaseFtpTree.__init__(self, tree)
        self.loadedpkgs={}
        self.marked4removal=[]
        self.marked4moving=[]
        self.pkgnames=[]
        self.__loadpkgnames()
        if loadall:
            for pkgname in self.pkgnames:
                self.loadedpkgs[pkgname]=Pkg(pkgname, self)
        # Tests:
        self.do_checkbuild=True

    def __getitem__(self, key):
        if self.loadedpkgs.has_key(key):
            return self.loadedpkgs[key]
        elif key in self.pkgnames:
            pkg=Pkg(key, self)
            self.loadedpkgs[key]=pkg
            return pkg
        else:
            raise KeyError, key

    def has_key(self, key):
        if key in self.pkgnames:
            return True
        else:
            return False

    def keys(self):
        return self.pkgnames

    def values(self):
        return self.loadedpkgs.values()

    def checktree(self, dsttree):
        self.__checkbuild(self.loadedpkgs.values())
        self.__checkarchs(dsttree, self.loadedpkgs.values())

    def testmove(self, dsttree):
        self.__checkbuild(self.marked4moving)
        self.__checkarchs(dsttree, self.marked4moving)
        
        self.__rmolderfromsrc(test=True)
        self.__rmotherfromdst(dsttree, test=True)

        for pkg in self.marked4moving:
            pkg.move(dsttree, test=True)

    def movepkgs(self, dsttree):
        if self.do_checkbuild:
            self.__checkbuild(self.marked4moving)
        bailoutonerror()
        self.__checkarchs(dsttree, self.marked4moving)
        bailoutonerror()
        self.__rmolderfromsrc()
        self.__rmotherfromdst(dsttree)

        for pkg in self.marked4moving:
            pkg.move(dsttree)

    def removepkgs(self):
        if self.do_checkbuild:
            self.__checkbuild(self.marked4removal)
        bailoutonerror()
        for pkg in self.marked4removal:
            pkg.remove()

    def mark4removal(self, wannabepkgs):
        self.__mark4something(wannabepkgs, Pkg.mark4removal)

    def mark4moving(self, wannabepkgs):
        self.__mark4something(wannabepkgs, Pkg.mark4moving)
        

    # Internal functions below

    def __loadpkgnames(self):
        def checkfiletype(name):
            if name[-13:]=='.src.rpm.info':
                return True
            else:
                return False
        list=filter(checkfiletype, os.listdir(self.basedir+'/SRPMS/.metadata'))
        self.pkgnames=map((lambda x: x[:-13]), list)

    def __mark4something(self, wannabepkgs, markfunction):
        def chopoffextension(pkg):
            found=pkg.find('.src.rpm')
            if found==-1:
                return pkg
            else:
                return pkg[:found]
        for wannabepkg in wannabepkgs:
            pkgname=chopoffextension(wannabepkg)
            if pkgname in self.pkgnames:
                if not pkgname in self.loadedpkgs.keys():
                    self.loadedpkgs[pkgname]=Pkg(pkgname, self)
                markfunction(self.loadedpkgs[pkgname])
            else:
                perror('%s not found in source tree' % pkgname)
        bailoutonerror()

    def __checkbuild(self, marked):
        f=urllib.urlopen(config.builderqueue)
        #f=open('queue.txt')
        requests={}
        reid=re.compile(r'^.*id=(.*) pri.*$')
        regb=re.compile(r'^group:.*$|builders:.*$', re.M)
        for i in re.findall(regb, f.read()):
            if i[0]=='g':
                id=reid.sub(r'\1', i)
                requests[id]=""
            elif i[0]=='b':
                requests[id]=requests[id]+i
        f.close()
        for pkg in marked:
            for bid in pkg.build.keys():
                if requests.has_key(bid) and not requests[bid].find('?') == -1:
                    pkg.error("(buildid %s) building not finished" % bid)

    def __checkarchs(self, dsttree, marked):
        for pkg in marked:
            if len(pkg.files.keys()) <= 1:
                pkg.error('has only src.rpm built')
                continue
            otherpkgnames=self.__find_other_pkgs(pkg, dsttree)
            if otherpkgnames: # check if we're not removing some archs
                curarchs=[]
                missingarchs=[]
                for somepkg in otherpkgnames:
                    curarchs.extend(Pkg(somepkg, dsttree).files.keys())
                for arch in curarchs:
                    if arch not in pkg.files.keys():
                        missingarchs.append(arch)
                if missingarchs:
                    pkg.error('moving would remove archs: %s' % missingarchs)
            else: # warn if a package isn't built for all archs
                if (config.separate_noarch and 'noarch' in pkg.files.keys() and
                                    len(pkg.files.keys())==2):
                    continue
                elif len(pkg.files.keys()) != len(config.ftp_archs)+1:
                    missingarchs=[]
                    for arch in config.ftp_archs:
                        if arch not in pkg.files.keys():
                            missingarchs.append(arch)
                    pkg.warning('not built for archs: %s' % missingarchs)

    def __rmolderfromsrc(self, test=False):
        for pkg in self.marked4moving:
            olderpkgnames=self.__find_older_pkgs(pkg)
            for i in olderpkgnames:
                Pkg(i, self).remove(test)

    def __rmotherfromdst(self, dsttree, test=False):
        for pkg in self.marked4moving:
            pkgnames=self.__find_other_pkgs(pkg, dsttree)
            for i in pkgnames:
                Pkg(i, dsttree).remove(test)

    # Used more than once filter functions

    def __find_other_pkgs(self, pkg, tree):
        escapedpkgname=pkg.name.replace('.', '\.').replace('+', '\+')
        ziewre=re.compile(escapedpkgname+'-[^-]*-[^-]*$')
        def filter_other_pkgs(x):
            if ziewre.match(x) and not x == pkg.nvr:
                return True
            else:
                return False
        return filter(filter_other_pkgs, tree.pkgnames)

    def __find_older_pkgs(self, pkg):
        def filter_older_pkgs(x):
            c=x.split('-')
            rc = rpm.labelCompare(('0', pkg.version, pkg.release),
                                                        ('0', c[-2], c[-1]))
            if rc == 1: # pkg > x
                return True
            else:
                return False
        return filter(filter_older_pkgs, self.__find_other_pkgs(pkg, self))