modules/ftptree.py

   1 # vi: encoding=utf-8 ts=8 sts=4 sw=4 et
   2
   3 import os, config, string, urllib, re, rpm
   4 from common import fileexists, noarchcachedir
   5 from baseftptree import BasePkg, BaseFtpTree
   6 errnum=0
   7
   8 SomeError="Oh no"
   9 quietmode=False
  10
  11 def bailoutonerror():
  12     if not errnum == 0:
  13         print "%d error(s) encountered... aborting" % errnum
  14         raise SomeError
  15
  16 def pinfo(msg):
  17     print 'INFO: ' + msg
  18
  19 def perror(msg):
  20     global errnum
  21     errnum=errnum+1
  22     print 'ERR: ' + msg
  23
  24 def pwarning(msg):
  25     print 'WARN: ' + msg
  26
  27 def rm(file):
  28     os.remove(file)
  29     #print 'rm: '+file
  30
  31 def mv(src, dst):
  32     try:
  33         os.rename(src, dst+'/'+src.split('/')[-1])
  34     except OSError, e:
  35         pinfo("os.rename(%s, %s): %s" % (src, dst+'/'+src.split('/')[-1], e))
  36         raise
  37
  38 class Pkg(BasePkg):
  39     def __init__(self, nvr, tree):
  40         BasePkg.__init__(self, nvr, tree)
  41         self.name=string.join(nvr.split('-')[:-2], '-')
  42         self.version=nvr.split('-')[-2]
  43         self.release=nvr.split('-')[-1]
  44         self.marked4removal=False
  45         self.marked4moving=False
  46         self.marked4movingpool=[]
  47         self.errors=[]
  48         self.warnings=[]
  49
  50     def __cmp__(self, pkg):
  51         if self.name > pkg.name:
  52             return 1
  53         elif self.name < pkg.name:
  54             return -1
  55         else:
  56             return rpm.labelCompare(('0', self.version, self.release),
  57                                     ('0', pkg.version, pkg.release))
  58
  59     def mark4moving(self):
  60         if not self.marked4moving:
  61             # Only one pkg in this pool can be marked for moving
  62             for pkg in self.marked4movingpool:
  63                 pkg.unmark4moving()
  64             self.tree.marked4moving.append(self)
  65             self.marked4moving=True
  66
  67     def unmark4moving(self):
  68         if self.marked4moving:
  69             self.tree.marked4moving.remove(self)
  70             self.marked4moving=False
  71
  72     def mark4removal(self):
  73         if not self.marked4removal:
  74             self.tree.marked4removal.append(self)
  75             self.marked4removal=True
  76
  77     def error(self, msg):
  78         self.errors.append(msg)
  79         if not quietmode:
  80             perror('%s %s' % (self.nvr, msg))
  81
  82     def warning(self, msg):
  83         self.warnings.append(msg)
  84         if not quietmode:
  85             pwarning('%s %s' % (self.nvr, msg))
  86
  87     def load(self, content=None):
  88         BasePkg.load(self, content)
  89         if self.info.has_key('move'):
  90             self.mark4moving()
  91
  92     def writeinfo(self):
  93         f=open(self.tree.basedir+'/SRPMS/.metadata/'+self.nvr+'.src.rpm.info', 'w')
  94         for bid in self.build.keys():
  95             f.write("info:build:%s:requester:%s\ninfo:build:%s:requester_email:%s\n" % (bid, self.build[bid].requester, bid, self.build[bid].requester_email))
  96         for key in self.info.keys():
  97             f.write("info:%s:%s\n" % (key, string.join(self.info[key], ':')))
  98         for arch in self.files.keys():
  99             for rpm in self.files[arch]:
 100                 f.write("file:%s:%s\n" % (arch, rpm))
 101
 102     def remove(self):
 103         for arch in self.files.keys():
 104             for rpm in self.files[arch]:
 105                 rm(self.tree.basedir+'/'+arch+'/RPMS/'+rpm)
 106                 if arch=='noarch':
 107                     if fileexists(noarchcachedir+rpm+'.filelist'):
 108                         rm(noarchcachedir+rpm+'.filelist')
 109                     if fileexists(noarchcachedir+rpm+'.reqlist'):
 110                         rm(noarchcachedir+rpm+'.reqlist')
 111         rm(self.tree.basedir+'/SRPMS/.metadata/'+self.nvr+'.src.rpm.info')
 112
 113     def move(self, dsttree):
 114         if dsttree.has_key(self.nvr):
 115             movedany=False
 116             for arch in self.files.keys():
 117                 if arch in dsttree[self.nvr].files.keys():
 118                     pinfo("Arch %s for %s is already present in dest tree; removing from srctree" % (arch, self.nvr))
 119                     for rpm in self.files[arch]:
 120                         rm(self.tree.basedir+'/'+arch+'/RPMS/'+rpm)
 121                 else:
 122                     movedany=True
 123                     dsttree[self.nvr].files[arch]=self.files[arch]
 124                     for rpm in self.files[arch]:
 125                         mv(self.tree.basedir+'/'+arch+'/RPMS/'+rpm, dsttree.basedir+'/'+arch+'/RPMS/')
 126             if movedany:
 127                 for bid in self.build.keys():
 128                     dsttree[self.nvr].build[bid]=self.build[bid]
 129                 dsttree[self.nvr].writeinfo()
 130             rm(self.tree.basedir+'/SRPMS/.metadata/'+self.nvr+'.src.rpm.info')
 131         else:
 132             for arch in self.files.keys():
 133                 for rpm in self.files[arch]:
 134                     mv(self.tree.basedir+'/'+arch+'/RPMS/'+rpm, dsttree.basedir+'/'+arch+'/RPMS/')
 135             mv(self.tree.basedir+'/SRPMS/.metadata/'+self.nvr+'.src.rpm.info', dsttree.basedir+'/SRPMS/.metadata/')
 136
 137
 138 class FtpTree(BaseFtpTree):
 139     def __init__(self, tree, loadall=False):
 140         BaseFtpTree.__init__(self, tree)
 141         self.loadedpkgs={}
 142         self.marked4removal=[]
 143         self.marked4moving=[]
 144         self.pkgnames=[]
 145         self.__loadpkgnames()
 146         if loadall:
 147             for pkgname in self.pkgnames:
 148                 self.loadedpkgs[pkgname]=Pkg(pkgname, self)
 149         # Tests:
 150         self.do_checkbuild=True
 151
 152     def __getitem__(self, key):
 153         if self.loadedpkgs.has_key(key):
 154             return self.loadedpkgs[key]
 155         elif key in self.pkgnames:
 156             pkg=Pkg(key, self)
 157             self.loadedpkgs[key]=pkg
 158             return pkg
 159         else:
 160             raise KeyError, key
 161
 162     def has_key(self, key):
 163         if key in self.pkgnames:
 164             return True
 165         else:
 166             return False
 167
 168     def keys(self):
 169         return self.pkgnames
 170
 171     def values(self):
 172         return self.loadedpkgs.values()
 173
 174     def checktree(self, dsttree):
 175         self.__checkbuild(self.loadedpkgs.values())
 176         self.__checkarchs(dsttree, self.loadedpkgs.values())
 177
 178     def testmove(self, dsttree):
 179         self.__checkbuild(self.marked4moving)
 180         self.__checkarchs(dsttree, self.marked4moving)
 181
 182     def movepkgs(self, dsttree):
 183         if self.do_checkbuild:
 184             self.__checkbuild(self.marked4moving)
 185         bailoutonerror()
 186         self.__checkarchs(dsttree, self.marked4moving)
 187         bailoutonerror()
 188         self.__rmolderfromsrc()
 189         self.__rmotherfromdst(dsttree)
 190
 191         for pkg in self.marked4moving:
 192             pkg.move(dsttree)
 193
 194     def removepkgs(self):
 195         if self.do_checkbuild:
 196             self.__checkbuild(self.marked4removal)
 197         bailoutonerror()
 198         for pkg in self.marked4removal:
 199             pkg.remove()
 200
 201     def mark4removal(self, wannabepkgs):
 202         self.__mark4something(wannabepkgs, Pkg.mark4removal)
 203
 204     def mark4moving(self, wannabepkgs):
 205         self.__mark4something(wannabepkgs, Pkg.mark4moving)
 206
 207
 208     # Internal functions below
 209
 210     def __loadpkgnames(self):
 211         def checkfiletype(name):
 212             if name[-13:]=='.src.rpm.info':
 213                 return True
 214             else:
 215                 return False
 216         list=filter(checkfiletype, os.listdir(self.basedir+'/SRPMS/.metadata'))
 217         self.pkgnames=map((lambda x: x[:-13]), list)
 218
 219     def __mark4something(self, wannabepkgs, markfunction):
 220         def chopoffextension(pkg):
 221             found=pkg.find('.src.rpm')
 222             if found==-1:
 223                 return pkg
 224             else:
 225                 return pkg[:found]
 226         for wannabepkg in wannabepkgs:
 227             pkgname=chopoffextension(wannabepkg)
 228             if pkgname in self.pkgnames:
 229                 if not pkgname in self.loadedpkgs.keys():
 230                     self.loadedpkgs[pkgname]=Pkg(pkgname, self)
 231                 markfunction(self.loadedpkgs[pkgname])
 232             else:
 233                 perror('%s not found in source tree' % pkgname)
 234         bailoutonerror()
 235
 236     def __checkbuild(self, marked):
 237         f=urllib.urlopen(config.builderqueue)
 238         #f=open('queue.txt')
 239         requests={}
 240         reid=re.compile(r'^.*id=(.*) pri.*$')
 241         regb=re.compile(r'^group:.*$|builders:.*$', re.M)
 242         for i in re.findall(regb, f.read()):
 243             if i[0]=='g':
 244                 id=reid.sub(r'\1', i)
 245                 requests[id]=""
 246             elif i[0]=='b':
 247                 requests[id]=requests[id]+i
 248         f.close()
 249         for pkg in marked:
 250             for bid in pkg.build.keys():
 251                 if requests.has_key(bid) and not requests[bid].find('?') == -1:
 252                     pkg.error("(buildid %s) building not finished" % bid)
 253
 254     def __checkarchs(self, dsttree, marked):
 255         for pkg in marked:
 256             if len(pkg.files.keys()) <= 1:
 257                 pkg.error('has only src.rpm built')
 258                 continue
 259             otherpkgnames=self.__find_other_pkgs(pkg, dsttree)
 260             if otherpkgnames: # check if we're not removing some archs
 261                 curarchs=[]
 262                 missingarchs=[]
 263                 for somepkg in otherpkgnames:
 264                     curarchs.extend(Pkg(somepkg, dsttree).files.keys())
 265                 for arch in curarchs:
 266                     if arch not in pkg.files.keys():
 267                         missingarchs.append(arch)
 268                 if missingarchs:
 269                     pkg.error('moving would remove archs: %s' % missingarchs)
 270             else: # warn if a package isn't built for all archs
 271                 if (config.separate_noarch and 'noarch' in pkg.files.keys() and
 272                                     len(pkg.files.keys())==2):
 273                     continue
 274                 elif len(pkg.files.keys()) != len(config.ftp_archs)+1:
 275                     missingarchs=[]
 276                     for arch in config.ftp_archs:
 277                         if arch not in pkg.files.keys():
 278                             missingarchs.append(arch)
 279                     pkg.warning('not built for archs: %s' % missingarchs)
 280
 281     def __rmolderfromsrc(self):
 282         for pkg in self.marked4moving:
 283             olderpkgnames=self.__find_older_pkgs(pkg)
 284             for i in olderpkgnames:
 285                 Pkg(i, self).remove()
 286
 287     def __rmotherfromdst(self, dsttree):
 288         for pkg in self.marked4moving:
 289             pkgnames=self.__find_other_pkgs(pkg, dsttree)
 290             for i in pkgnames:
 291                 Pkg(i, dsttree).remove()
 292
 293     # Used more than once filter functions
 294
 295     def __find_other_pkgs(self, pkg, tree):
 296         escapedpkgname=pkg.name.replace('.', '\.').replace('+', '\+')
 297         ziewre=re.compile(escapedpkgname+'-[^-]*-[^-]*$')
 298         def filter_other_pkgs(x):
 299             if ziewre.match(x) and not x == pkg.nvr:
 300                 return True
 301             else:
 302                 return False
 303         return filter(filter_other_pkgs, tree.pkgnames)
 304
 305     def __find_older_pkgs(self, pkg):
 306         def filter_older_pkgs(x):
 307             c=x.split('-')
 308             rc = rpm.labelCompare(('0', pkg.version, pkg.release),
 309                                                         ('0', c[-2], c[-1]))
 310             if rc == 1: # pkg > x
 311                 return True
 312             else:
 313                 return False
 314         return filter(filter_older_pkgs, self.__find_other_pkgs(pkg, self))
 315