-#!/usr/bin/env python
# vi: encoding=utf-8 ts=8 sts=4 sw=4 et
-import sys, os, config, string, urllib, re
+import os, config, string, urllib, re, rpm
+from common import fileexists, noarchcachedir
+from baseftptree import BasePkg, BaseFtpTree
errnum=0
+quietmode=False
+
+class SomeError(Exception):
+ def __init__(self):
+ return
+
+ def __str__(self):
+ print "","An Error occured!"
+
def bailoutonerror():
if not errnum == 0:
print "%d error(s) encountered... aborting" % errnum
- sys.exit(1)
+ raise SomeError
+
+def pinfo(msg):
+ print 'INFO: ' + msg
def perror(msg):
global errnum
errnum=errnum+1
- print msg
+ print 'ERR: ' + msg
-def rm(file):
- print 'rm: '+file
+def pwarning(msg):
+ print 'WARN: ' + msg
-def mv(src, dst):
- print "mv: %s %s" % (src, dst)
+def rm(file, test=False):
+ if test:
+ if not os.path.exists(file):
+ pinfo("TEST os.remove(%s): file doesn't exists" % file)
+ else:
+ try:
+ os.remove(file)
+ except OSError, e:
+ pinfo("os.remove(%s): %s" % (file, e))
+ #raise
-class Build:
- def __init__(self):
- self.requester=''
- self.requester_email=''
-
-class Pkg:
- def __init__(self, name, tree):
- self.file={}
- self.info={}
- self.build={}
- self.tree=tree
- self.name=name
- self.loaded=False
- self.marked=False
- self.load()
-
- def __repr__(self):
- return self.name
-
- def mark(self):
- if not self.marked:
- self.tree.markedpkgs.append(self)
-
- def load(self):
- f=open(self.tree.basedir+'/SRPMS/.metadata/'+self.name+'.src.rpm.info', 'r')
- for entry in f.readlines():
- i=string.split(string.strip(entry), ':')
- if i[0] == 'info':
- if len(i)==3:
- self.info[i[1]]=i[2]
- elif i[1]=='build':
- if not self.build.has_key(i[2]):
- self.build[i[2]]=Build()
- if i[3]=='requester':
- self.build[i[2]].requester=i[4]
- elif i[3]=='requester_email':
- self.build[i[2]].requester_email=i[4]
- else:
- self.info[i[1]]=i[2:]
- elif i[0] == 'file':
- if not self.file.has_key(i[1]):
- self.file[i[1]]=[]
- self.file[i[1]].append(i[2])
- f.close()
- self.loaded=True
+def mv(src, dst, test=False):
+ fsrc = src
+ fdst = dst+'/'+src.split('/')[-1]
+ if test:
+ if not os.path.exists(src):
+ pinfo("TEST os.rename(%s, %s): source doesn't exists" % (fsrc, fdst))
+ else:
+ try:
+ os.rename(fsrc, fdst)
+ except OSError, e:
+ pinfo("os.rename(%s, %s): %s" % (fsrc, fdst, e))
+ raise
+
+class Pkg(BasePkg):
+ def __init__(self, nvr, tree):
+ BasePkg.__init__(self, nvr, tree)
+ self.name = string.join(nvr.split('-')[:-2], '-')
+ self.version = nvr.split('-')[-2]
+ self.release = nvr.split('-')[-1]
+ self.marked4removal = False
+ self.marked4moving = False
+ self.marked4movingpool = []
+ self.errors = []
+ self.warnings = []
+
+ def __cmp__(self, pkg):
+ if self.name > pkg.name:
+ return 1
+ elif self.name < pkg.name:
+ return -1
+ else:
+ return rpm.labelCompare(('0', self.version, self.release),
+ ('0', pkg.version, pkg.release))
+
+
+ # unfortunately can't do new Pkg(NVR), and have no "tree" in this pkg context
+ # so this static function
+ def is_debuginfo(self, nvr):
+ """
+ returns true if NVR is debuginfo package and separate debuginfo is enabled
+ """
+ if not config.separate_debuginfo:
+ return False
+ pkg = nvr.split('-')[:-2]
+ return pkg[-1] == 'debuginfo'
+
+ def mark4moving(self):
+ if not self.marked4moving:
+ # Only one pkg in this pool can be marked for moving
+ for pkg in self.marked4movingpool:
+ pkg.unmark4moving()
+ self.tree.marked4moving.append(self)
+ self.marked4moving=True
+
+ def unmark4moving(self):
+ if self.marked4moving:
+ self.tree.marked4moving.remove(self)
+ self.marked4moving=False
+
+ def mark4removal(self):
+ if not self.marked4removal:
+ self.tree.marked4removal.append(self)
+ self.marked4removal=True
+
+ def error(self, msg):
+ self.errors.append(msg)
+ if not quietmode:
+ perror('%s %s' % (self.nvr, msg))
+
+ def warning(self, msg):
+ self.warnings.append(msg)
+ if not quietmode:
+ pwarning('%s %s' % (self.nvr, msg))
+
+ def load(self, content=None):
+ BasePkg.load(self, content)
if self.info.has_key('move'):
- self.mark()
+ self.mark4moving()
def writeinfo(self):
- f=open(self.tree.basedir+'/SRPMS/.metadata/'+name+'.src.rpm.info', 'w')
+ f = open(self.tree.basedir+'/SRPMS/.metadata/'+self.nvr+'.src.rpm.info', 'w')
for bid in self.build.keys():
f.write("info:build:%s:requester:%s\ninfo:build:%s:requester_email:%s\n" % (bid, self.build[bid].requester, bid, self.build[bid].requester_email))
for key in self.info.keys():
f.write("info:%s:%s\n" % (key, string.join(self.info[key], ':')))
- for arch in self.file.keys():
- for rpm in self.file[arch]:
+ for arch in self.files.keys():
+ for rpm in self.files[arch]:
f.write("file:%s:%s\n" % (arch, rpm))
- def remove(self):
- for arch in self.file.keys():
- for rpm in self.file[arch]:
- rm(self.tree.basedir+'/'+arch+'/RPMS/'+rpm)
- rm(self.tree.basedir+'/SRPMS/.metadata/'+self.name+'.src.rpm.info')
-
- def move(self, dsttree):
- if dsttree.has_key(self.name):
- movedany=False
- for arch in self.file.keys():
- if arch in dsttree[self.name].file.keys():
- for rpm in self.file[arch]:
- rm(self.tree.basedir+'/'+arch+'/RPMS/'+rpm)
+ def remove(self, test = False):
+ """
+ Remove package from ftp
+ """
+ for arch in self.files.keys():
+ for rpm in self.files[arch]:
+ if self.is_debuginfo(rpm):
+ rm(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, test)
+ else:
+ rm(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, test)
+ if arch == 'noarch':
+ if fileexists(noarchcachedir + rpm + '.filelist'):
+ rm(noarchcachedir + rpm + '.filelist', test)
+ if fileexists(noarchcachedir + rpm + '.reqlist'):
+ rm(noarchcachedir + rpm + '.reqlist', test)
+ rm(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', test)
+
+ def rpmfiles(self):
+ """
+ Return rpm files related to this package
+ """
+ files = []
+ for arch, rpms in self.files.items():
+ for nvr in rpms:
+ if self.is_debuginfo(nvr):
+ files.append(self.tree.basedir + '/' + arch + '/debuginfo/' + nvr)
else:
- movedany=True
- dsttree[self.name].file[arch]=self.file[arch]
- for rpm in self.file[arch]:
- mv(self.tree.basedir+'/'+arch+'/RPMS/'+rpm, dsttree.basedir+'/'+arch+'/RPMS/')
- if movedany:
+ files.append(self.tree.basedir + '/' + arch + '/RPMS/' + nvr)
+ return files
+
+ def move(self, dsttree, test=False):
+ if dsttree.has_key(self.nvr):
+ movedany = False
+ for arch in self.files.keys():
+ if arch in dsttree[self.nvr].files.keys():
+ msg = ""
+ if test:
+ msg = "TEST "
+ pinfo("%sArch %s for %s is already present in dest tree; removing from srctree" % (msg, arch, self.nvr))
+ for rpm in self.files[arch]:
+ if self.is_debuginfo(rpm):
+ rm(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, test)
+ else:
+ rm(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, test)
+ else:
+ movedany = True
+ dsttree[self.nvr].files[arch] = self.files[arch]
+ for rpm in self.files[arch]:
+ if self.is_debuginfo(rpm):
+ mv(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, dsttree.basedir + '/' + arch + '/debuginfo/', test)
+ else:
+ mv(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, dsttree.basedir + '/' + arch + '/RPMS/', test)
+ if not test and movedany:
for bid in self.build.keys():
- dsttree[self.name].build[bid]=self.build[bid]
- dsttree[self.name].writeinfo()
- rm(self.tree.basedir+'/SRPMS/.metadata/'+self.name+'.src.rpm.info')
+ dsttree[self.nvr].build[bid] = self.build[bid]
+ dsttree[self.nvr].writeinfo()
+ rm(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', test)
else:
- for arch in self.file.keys():
- for rpm in self.file[arch]:
- mv(self.tree.basedir+'/'+arch+'/RPMS/'+rpm, dsttree.basedir+'/'+arch+'/RPMS/')
- mv(self.tree.basedir+'/SRPMS/.metadata/'+self.name+'.src.rpm.info', dsttree.basedir+'/SRPMS/.metadata/')
+ # move files
+ for arch in self.files.keys():
+ for rpm in self.files[arch]:
+ if self.is_debuginfo(rpm):
+ mv(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, dsttree.basedir + '/' + arch + '/debuginfo/', test)
+ else:
+ mv(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, dsttree.basedir + '/' + arch + '/RPMS/', test)
+ # move metadata
+ mv(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', dsttree.basedir + '/SRPMS/.metadata/', test)
-class FtpTree:
+class FtpTree(BaseFtpTree):
def __init__(self, tree, loadall=False):
- self.basedir=config.value['ftp_dir']+'/'+tree
- self.loadedpkgs={}
- self.markedpkgs=[]
- def filter_pkgs(pkg):
- if not pkg[-13:]=='.src.rpm.info':
- return None
- else:
- return pkg[:-13]
- self.pkgnames=map(filter_pkgs, os.listdir(self.basedir+'/SRPMS/.metadata/'))
+ BaseFtpTree.__init__(self, tree)
+ self.loadedpkgs = {}
+ self.marked4removal = []
+ self.marked4moving = []
+ self.pkgnames = []
+ self.__loadpkgnames()
if loadall:
- for i in self.pkgnames:
- self.loadedpkgs[i]=Pkg(i, self)
+ for pkgname in self.pkgnames:
+ self.loadedpkgs[pkgname] = Pkg(pkgname, self)
+ # Tests:
+ self.do_checkbuild = True
+
def __getitem__(self, key):
if self.loadedpkgs.has_key(key):
return self.loadedpkgs[key]
return pkg
else:
raise KeyError, key
+
def has_key(self, key):
if key in self.pkgnames:
return True
else:
return False
+ def keys(self):
+ return self.pkgnames
+
+ def values(self):
+ return self.loadedpkgs.values()
+
+ def checktree(self, dsttree):
+ self.__checkbuild(self.loadedpkgs.values())
+ self.__checkarchs(dsttree, self.loadedpkgs.values())
+
+ def testmove(self, dsttree):
+ self.__checkbuild(self.marked4moving)
+ self.__checkarchs(dsttree, self.marked4moving)
+
+ self.__rmolderfromsrc(test = True)
+ self.__rmotherfromdst(dsttree, test = True)
+
+ for pkg in self.marked4moving:
+ pkg.move(dsttree, test = True)
+
def movepkgs(self, dsttree):
- self.__checkbuild()
+ if self.do_checkbuild:
+ self.__checkbuild(self.marked4moving)
bailoutonerror()
- self.__checkarchs(dsttree)
+ self.__checkarchs(dsttree, self.marked4moving)
bailoutonerror()
self.__rmolderfromsrc()
self.__rmotherfromdst(dsttree)
- for pkg in self.markedpkgs:
+ for pkg in self.marked4moving:
pkg.move(dsttree)
- def markpkgs(self, wannabepkgs):
+ def rpmfiles(self):
+ if self.do_checkbuild:
+ self.__checkbuild(self.marked4moving)
+ bailoutonerror()
+
+ files = []
+ for pkg in self.marked4moving:
+ files += pkg.rpmfiles()
+ return files
+
+ def removepkgs(self):
+ if self.do_checkbuild:
+ self.__checkbuild(self.marked4removal)
+ bailoutonerror()
+ for pkg in self.marked4removal:
+ pkg.remove()
+
+ def mark4removal(self, wannabepkgs):
+ self.__mark4something(wannabepkgs, Pkg.mark4removal)
+
+ def mark4moving(self, wannabepkgs):
+ self.__mark4something(wannabepkgs, Pkg.mark4moving)
+
+
+ # Internal functions below
+
+ def __loadpkgnames(self):
+ def checkfiletype(name):
+ if name[-13:]=='.src.rpm.info':
+ return True
+ else:
+ return False
+ list = filter(checkfiletype, os.listdir(self.basedir+'/SRPMS/.metadata'))
+ self.pkgnames = map((lambda x: x[:-13]), list)
+
+ def __mark4something(self, wannabepkgs, markfunction):
def chopoffextension(pkg):
found=pkg.find('.src.rpm')
if found==-1:
if pkgname in self.pkgnames:
if not pkgname in self.loadedpkgs.keys():
self.loadedpkgs[pkgname]=Pkg(pkgname, self)
- self.loadedpkgs[pkgname].mark()
+ markfunction(self.loadedpkgs[pkgname])
else:
- perror(pkgname+" was not found in source tree")
+ perror('%s not found in source tree' % pkgname)
bailoutonerror()
- def __checkbuild(self):
- #f=urllib.urlopen('http://ep09.pld-linux.org/~builderth/queue.txt')
- f=open('queue.txt')
- requests={}
- reid=re.compile(r'^.*id=(.*) pri.*$')
- regb=re.compile(r'^group:.*$|builders:.*$', re.M)
+
+ def __checkbuild(self, marked):
+ """
+ Checks queue file if all arches are built
+
+ Reads config.builderqueue to grab the info
+ """
+ f = urllib.urlopen(config.builderqueue)
+ requests = {}
+ reid = re.compile(r'^.*id=(.*) pri.*$')
+ regb = re.compile(r'^group:.*$|builders:.*$', re.M)
for i in re.findall(regb, f.read()):
- if i[0]=='g':
- id=reid.sub(r'\1', i)
- requests[id]=""
+ if i[0] == 'g':
+ id = reid.sub(r'\1', i)
+ requests[id] = ""
elif i[0]=='b':
- requests[id]=requests[id]+i
+ requests[id] = requests[id] + i
f.close()
- for pkg in self.markedpkgs:
+
+ for pkg in marked:
for bid in pkg.build.keys():
if requests.has_key(bid) and not requests[bid].find('?') == -1:
- perror("Building of package %s (buildid %s) not finished" % (pkg,bid))
-
- def __checkarchs(self, dsttree):
- def find_other_pkgs(pkg, tree):
- ziewre=re.compile(string.join(pkg.name.split('-')[:-2], '-')+'-[^-]*-[^-]*$')
- def filter_other_pkgs(x):
- if ziewre.match(x) and not x == pkg.name:
- return x
- else:
- return None
- return filter(filter_other_pkgs, tree.pkgnames)
- self.find_other_pkgs=find_other_pkgs
- for pkg in self.markedpkgs:
- otherpkgnames=find_other_pkgs(pkg, dsttree)
- curarchs=[]
- missingarchs=[]
- for somepkg in otherpkgnames:
- curarchs.extend(Pkg(somepkg, dsttree).file.keys())
- for arch in curarchs:
- if arch not in pkg.file.keys():
- missingarchs.append(arch)
- if missingarchs:
- perror('Moving pkg %s would remove archs: %s' % (pkg, missingarchs))
-
- def __rmolderfromsrc(self):
- def find_older_pkgs(pkg):
- def filter_older_pkgs(x):
- ziew=x.split('-')
- cur=pkg.name.split('-')
- if ziew[-2]<cur[-2]:
- return x
- elif ziew[-2]==cur[-2]:
- if ziew[-1]<cur[-1]:
- return x
- else:
- return None
- else:
- return None
- return filter(filter_older_pkgs, self.find_other_pkgs(pkg, self))
+ pkg.error("(buildid %s) building not finished" % bid)
+
+ def __checkarchs(self, dsttree, marked):
+ """
+ Checks marked pkgs it is built on all archs.
+ """
+ for pkg in marked:
+ if len(pkg.files.keys()) <= 1:
+ pkg.error('has only src.rpm built')
+ continue
+ otherpkgnames = self.__find_other_pkgs(pkg, dsttree)
+
+ # check if we're not removing some archs
+ if otherpkgnames:
+ curarchs = []
+ missingarchs = []
+ for somepkg in otherpkgnames:
+ curarchs.extend(Pkg(somepkg, dsttree).files.keys())
+ for arch in curarchs:
+ if arch not in pkg.files.keys():
+ missingarchs.append(arch)
+ if missingarchs:
+ pkg.error('moving would remove archs: %s' % missingarchs)
+ else:
+ # warn if a package isn't built for all archs
+ if (config.separate_noarch and 'noarch' in pkg.files.keys() and len(pkg.files.keys()) == 2):
+ continue
+ elif len(pkg.files.keys()) != len(config.ftp_archs) + 1:
+ missingarchs = []
+ for arch in config.ftp_archs:
+ if arch not in pkg.files.keys():
+ missingarchs.append(arch)
+ pkg.warning('not built for archs: %s' % missingarchs)
- for pkg in self.markedpkgs:
- olderpkgnames=find_older_pkgs(pkg)
+ def __rmolderfromsrc(self, test=False):
+ for pkg in self.marked4moving:
+ olderpkgnames = self.__find_older_pkgs(pkg)
for i in olderpkgnames:
- Pkg(i, self).remove()
+ Pkg(i, self).remove(test)
- def __rmotherfromdst(self, dsttree):
- for pkg in self.markedpkgs:
- pkgnames=self.find_other_pkgs(pkg, dsttree)
+ def __rmotherfromdst(self, dsttree, test = False):
+ for pkg in self.marked4moving:
+ pkgnames = self.__find_other_pkgs(pkg, dsttree)
for i in pkgnames:
- Pkg(i, dsttree).remove()
+ Pkg(i, dsttree).remove(test)
+ # Used more than once filter functions
+ def __find_other_pkgs(self, pkg, tree):
+ escapedpkgname = pkg.name.replace('.', '\.').replace('+', '\+')
+ ziewre = re.compile(escapedpkgname + '-[^-]*-[^-]*$')
+ def filter_other_pkgs(x):
+ if ziewre.match(x) and not x == pkg.nvr:
+ return True
+ else:
+ return False
+ return filter(filter_other_pkgs, tree.pkgnames)
+
+ def __find_older_pkgs(self, pkg):
+ def filter_older_pkgs(x):
+ c = x.split('-')
+ rc = rpm.labelCompare(('0', pkg.version, pkg.release),
+ ('0', c[-2], c[-1]))
+ if rc == 1: # pkg > x
+ return True
+ else:
+ return False
+ return filter(filter_older_pkgs, self.__find_other_pkgs(pkg, self))