modules/ftptree.py

   1 # vi: encoding=utf-8 ts=8 sts=4 sw=4 et
   2
   3 import os, config, string, urllib, re, rpm
   4 from common import fileexists, noarchcachedir
   5 from baseftptree import BasePkg, BaseFtpTree
   6 errnum=0
   7
   8 quietmode=False
   9
  10 import exceptions
  11 class SomeException(exceptions.Exception):
  12     def __init__(self):
  13         return
  14
  15     def __str__(self):
  16         print "","An Error occured!"
  17
  18 def bailoutonerror():
  19     if not errnum == 0:
  20         print "%d error(s) encountered... aborting" % errnum
  21         raise SomeException
  22
  23 def pinfo(msg):
  24     print 'INFO: ' + msg
  25
  26 def perror(msg):
  27     global errnum
  28     errnum=errnum+1
  29     print 'ERR: ' + msg
  30
  31 def pwarning(msg):
  32     print 'WARN: ' + msg
  33
  34 def rm(file, test=False):
  35     if test:
  36         if not os.path.exists(file):
  37             pinfo("TEST os.remove(%s): file doesn't exists" % file)
  38     else:
  39         os.remove(file)
  40         #print 'rm: '+file
  41
  42 def mv(src, dst, test=False):
  43     fsrc = src
  44     fdst = dst+'/'+src.split('/')[-1]
  45     if test:
  46         if not os.path.exists(src):
  47             pinfo("TEST os.rename(%s, %s): source doesn't exists" % (fsrc, fdst))
  48     else:
  49         try:
  50             os.rename(fsrc, fdst)
  51         except OSError, e:
  52             pinfo("os.rename(%s, %s): %s" % (fsrc, fdst, e))
  53             raise
  54
  55 class Pkg(BasePkg):
  56     def __init__(self, nvr, tree):
  57         BasePkg.__init__(self, nvr, tree)
  58         self.name=string.join(nvr.split('-')[:-2], '-')
  59         self.version=nvr.split('-')[-2]
  60         self.release=nvr.split('-')[-1]
  61         self.marked4removal=False
  62         self.marked4moving=False
  63         self.marked4movingpool=[]
  64         self.errors=[]
  65         self.warnings=[]
  66
  67     def __cmp__(self, pkg):
  68         if self.name > pkg.name:
  69             return 1
  70         elif self.name < pkg.name:
  71             return -1
  72         else:
  73             return rpm.labelCompare(('0', self.version, self.release),
  74                                     ('0', pkg.version, pkg.release))
  75
  76     def mark4moving(self):
  77         if not self.marked4moving:
  78             # Only one pkg in this pool can be marked for moving
  79             for pkg in self.marked4movingpool:
  80                 pkg.unmark4moving()
  81             self.tree.marked4moving.append(self)
  82             self.marked4moving=True
  83
  84     def unmark4moving(self):
  85         if self.marked4moving:
  86             self.tree.marked4moving.remove(self)
  87             self.marked4moving=False
  88
  89     def mark4removal(self):
  90         if not self.marked4removal:
  91             self.tree.marked4removal.append(self)
  92             self.marked4removal=True
  93
  94     def error(self, msg):
  95         self.errors.append(msg)
  96         if not quietmode:
  97             perror('%s %s' % (self.nvr, msg))
  98
  99     def warning(self, msg):
 100         self.warnings.append(msg)
 101         if not quietmode:
 102             pwarning('%s %s' % (self.nvr, msg))
 103
 104     def load(self, content=None):
 105         BasePkg.load(self, content)
 106         if self.info.has_key('move'):
 107             self.mark4moving()
 108
 109     def writeinfo(self):
 110         f=open(self.tree.basedir+'/SRPMS/.metadata/'+self.nvr+'.src.rpm.info', 'w')
 111         for bid in self.build.keys():
 112             f.write("info:build:%s:requester:%s\ninfo:build:%s:requester_email:%s\n" % (bid, self.build[bid].requester, bid, self.build[bid].requester_email))
 113         for key in self.info.keys():
 114             f.write("info:%s:%s\n" % (key, string.join(self.info[key], ':')))
 115         for arch in self.files.keys():
 116             for rpm in self.files[arch]:
 117                 f.write("file:%s:%s\n" % (arch, rpm))
 118
 119     def remove(self, test=False):
 120         for arch in self.files.keys():
 121             for rpm in self.files[arch]:
 122                 rm(self.tree.basedir+'/'+arch+'/RPMS/'+rpm, test)
 123                 if arch=='noarch':
 124                     if fileexists(noarchcachedir+rpm+'.filelist'):
 125                         rm(noarchcachedir+rpm+'.filelist', test)
 126                     if fileexists(noarchcachedir+rpm+'.reqlist'):
 127                         rm(noarchcachedir+rpm+'.reqlist', test)
 128         rm(self.tree.basedir+'/SRPMS/.metadata/'+self.nvr+'.src.rpm.info', test)
 129
 130     def move(self, dsttree, test=False):
 131         if dsttree.has_key(self.nvr):
 132             movedany=False
 133             for arch in self.files.keys():
 134                 if arch in dsttree[self.nvr].files.keys():
 135                     msg = ""
 136                     if test:
 137                         msg = "TEST "
 138                     pinfo("%sArch %s for %s is already present in dest tree; removing from srctree" % (msg, arch, self.nvr))
 139                     for rpm in self.files[arch]:
 140                         rm(self.tree.basedir+'/'+arch+'/RPMS/'+rpm, test)
 141                 else:
 142                     movedany=True
 143                     dsttree[self.nvr].files[arch]=self.files[arch]
 144                     for rpm in self.files[arch]:
 145                         mv(self.tree.basedir+'/'+arch+'/RPMS/'+rpm, dsttree.basedir+'/'+arch+'/RPMS/', test)
 146             if not test and  movedany:
 147                 for bid in self.build.keys():
 148                     dsttree[self.nvr].build[bid]=self.build[bid]
 149                 dsttree[self.nvr].writeinfo()
 150             rm(self.tree.basedir+'/SRPMS/.metadata/'+self.nvr+'.src.rpm.info', test)
 151         else:
 152             for arch in self.files.keys():
 153                 for rpm in self.files[arch]:
 154                     mv(self.tree.basedir+'/'+arch+'/RPMS/'+rpm, dsttree.basedir+'/'+arch+'/RPMS/', test)
 155             mv(self.tree.basedir+'/SRPMS/.metadata/'+self.nvr+'.src.rpm.info', dsttree.basedir+'/SRPMS/.metadata/', test)
 156
 157
 158 class FtpTree(BaseFtpTree):
 159     def __init__(self, tree, loadall=False):
 160         BaseFtpTree.__init__(self, tree)
 161         self.loadedpkgs={}
 162         self.marked4removal=[]
 163         self.marked4moving=[]
 164         self.pkgnames=[]
 165         self.__loadpkgnames()
 166         if loadall:
 167             for pkgname in self.pkgnames:
 168                 self.loadedpkgs[pkgname]=Pkg(pkgname, self)
 169         # Tests:
 170         self.do_checkbuild=True
 171
 172     def __getitem__(self, key):
 173         if self.loadedpkgs.has_key(key):
 174             return self.loadedpkgs[key]
 175         elif key in self.pkgnames:
 176             pkg=Pkg(key, self)
 177             self.loadedpkgs[key]=pkg
 178             return pkg
 179         else:
 180             raise KeyError, key
 181
 182     def has_key(self, key):
 183         if key in self.pkgnames:
 184             return True
 185         else:
 186             return False
 187
 188     def keys(self):
 189         return self.pkgnames
 190
 191     def values(self):
 192         return self.loadedpkgs.values()
 193
 194     def checktree(self, dsttree):
 195         self.__checkbuild(self.loadedpkgs.values())
 196         self.__checkarchs(dsttree, self.loadedpkgs.values())
 197
 198     def testmove(self, dsttree):
 199         self.__checkbuild(self.marked4moving)
 200         self.__checkarchs(dsttree, self.marked4moving)
 201
 202         self.__rmolderfromsrc(test=True)
 203         self.__rmotherfromdst(dsttree, test=True)
 204
 205         for pkg in self.marked4moving:
 206             pkg.move(dsttree, test=True)
 207
 208     def movepkgs(self, dsttree):
 209         if self.do_checkbuild:
 210             self.__checkbuild(self.marked4moving)
 211         bailoutonerror()
 212         self.__checkarchs(dsttree, self.marked4moving)
 213         bailoutonerror()
 214         self.__rmolderfromsrc()
 215         self.__rmotherfromdst(dsttree)
 216
 217         for pkg in self.marked4moving:
 218             pkg.move(dsttree)
 219
 220     def removepkgs(self):
 221         if self.do_checkbuild:
 222             self.__checkbuild(self.marked4removal)
 223         bailoutonerror()
 224         for pkg in self.marked4removal:
 225             pkg.remove()
 226
 227     def mark4removal(self, wannabepkgs):
 228         self.__mark4something(wannabepkgs, Pkg.mark4removal)
 229
 230     def mark4moving(self, wannabepkgs):
 231         self.__mark4something(wannabepkgs, Pkg.mark4moving)
 232
 233
 234     # Internal functions below
 235
 236     def __loadpkgnames(self):
 237         def checkfiletype(name):
 238             if name[-13:]=='.src.rpm.info':
 239                 return True
 240             else:
 241                 return False
 242         list=filter(checkfiletype, os.listdir(self.basedir+'/SRPMS/.metadata'))
 243         self.pkgnames=map((lambda x: x[:-13]), list)
 244
 245     def __mark4something(self, wannabepkgs, markfunction):
 246         def chopoffextension(pkg):
 247             found=pkg.find('.src.rpm')
 248             if found==-1:
 249                 return pkg
 250             else:
 251                 return pkg[:found]
 252         for wannabepkg in wannabepkgs:
 253             pkgname=chopoffextension(wannabepkg)
 254             if pkgname in self.pkgnames:
 255                 if not pkgname in self.loadedpkgs.keys():
 256                     self.loadedpkgs[pkgname]=Pkg(pkgname, self)
 257                 markfunction(self.loadedpkgs[pkgname])
 258             else:
 259                 perror('%s not found in source tree' % pkgname)
 260         bailoutonerror()
 261
 262     def __checkbuild(self, marked):
 263         f=urllib.urlopen(config.builderqueue)
 264         #f=open('queue.txt')
 265         requests={}
 266         reid=re.compile(r'^.*id=(.*) pri.*$')
 267         regb=re.compile(r'^group:.*$|builders:.*$', re.M)
 268         for i in re.findall(regb, f.read()):
 269             if i[0]=='g':
 270                 id=reid.sub(r'\1', i)
 271                 requests[id]=""
 272             elif i[0]=='b':
 273                 requests[id]=requests[id]+i
 274         f.close()
 275         for pkg in marked:
 276             for bid in pkg.build.keys():
 277                 if requests.has_key(bid) and not requests[bid].find('?') == -1:
 278                     pkg.error("(buildid %s) building not finished" % bid)
 279
 280     def __checkarchs(self, dsttree, marked):
 281         for pkg in marked:
 282             if len(pkg.files.keys()) <= 1:
 283                 pkg.error('has only src.rpm built')
 284                 continue
 285             otherpkgnames=self.__find_other_pkgs(pkg, dsttree)
 286             if otherpkgnames: # check if we're not removing some archs
 287                 curarchs=[]
 288                 missingarchs=[]
 289                 for somepkg in otherpkgnames:
 290                     curarchs.extend(Pkg(somepkg, dsttree).files.keys())
 291                 for arch in curarchs:
 292                     if arch not in pkg.files.keys():
 293                         missingarchs.append(arch)
 294                 if missingarchs:
 295                     pkg.error('moving would remove archs: %s' % missingarchs)
 296             else: # warn if a package isn't built for all archs
 297                 if (config.separate_noarch and 'noarch' in pkg.files.keys() and
 298                                     len(pkg.files.keys())==2):
 299                     continue
 300                 elif len(pkg.files.keys()) != len(config.ftp_archs)+1:
 301                     missingarchs=[]
 302                     for arch in config.ftp_archs:
 303                         if arch not in pkg.files.keys():
 304                             missingarchs.append(arch)
 305                     pkg.warning('not built for archs: %s' % missingarchs)
 306
 307     def __rmolderfromsrc(self, test=False):
 308         for pkg in self.marked4moving:
 309             olderpkgnames=self.__find_older_pkgs(pkg)
 310             for i in olderpkgnames:
 311                 Pkg(i, self).remove(test)
 312
 313     def __rmotherfromdst(self, dsttree, test=False):
 314         for pkg in self.marked4moving:
 315             pkgnames=self.__find_other_pkgs(pkg, dsttree)
 316             for i in pkgnames:
 317                 Pkg(i, dsttree).remove(test)
 318
 319     # Used more than once filter functions
 320
 321     def __find_other_pkgs(self, pkg, tree):
 322         escapedpkgname=pkg.name.replace('.', '\.').replace('+', '\+')
 323         ziewre=re.compile(escapedpkgname+'-[^-]*-[^-]*$')
 324         def filter_other_pkgs(x):
 325             if ziewre.match(x) and not x == pkg.nvr:
 326                 return True
 327             else:
 328                 return False
 329         return filter(filter_other_pkgs, tree.pkgnames)
 330
 331     def __find_older_pkgs(self, pkg):
 332         def filter_older_pkgs(x):
 333             c=x.split('-')
 334             rc = rpm.labelCompare(('0', pkg.version, pkg.release),
 335                                                         ('0', c[-2], c[-1]))
 336             if rc == 1: # pkg > x
 337                 return True
 338             else:
 339                 return False
 340         return filter(filter_older_pkgs, self.__find_other_pkgs(pkg, self))
 341