modules/ftptree.py

   1 # vi: encoding=utf-8 ts=8 sts=4 sw=4 et
   2
   3 import os, config, string, urllib, re, rpm
   4 from common import fileexists, noarchcachedir
   5 from baseftptree import BasePkg, BaseFtpTree
   6 errnum=0
   7
   8 quietmode=False
   9
  10 class SomeError(Exception):
  11     def __init__(self):
  12         return
  13
  14     def __str__(self):
  15         print "","An Error occured!"
  16
  17 def bailoutonerror():
  18     if not errnum == 0:
  19         print "%d error(s) encountered... aborting" % errnum
  20         raise SomeError
  21
  22 def pinfo(msg):
  23     print 'INFO: ' + msg
  24
  25 def perror(msg):
  26     global errnum
  27     errnum=errnum+1
  28     print 'ERR: ' + msg
  29
  30 def pwarning(msg):
  31     print 'WARN: ' + msg
  32
  33 def rm(file, test=False):
  34     if test:
  35         if not os.path.exists(file):
  36             pinfo("TEST os.remove(%s): file doesn't exists" % file)
  37     else:
  38         try:
  39             os.remove(file)
  40         except OSError, e:
  41             pinfo("os.remove(%s): %s" % (file, e))
  42             raise
  43
  44 def mv(src, dst, test=False):
  45     fsrc = src
  46     fdst = dst+'/'+src.split('/')[-1]
  47     if test:
  48         if not os.path.exists(src):
  49             pinfo("TEST os.rename(%s, %s): source doesn't exists" % (fsrc, fdst))
  50     else:
  51         try:
  52             os.rename(fsrc, fdst)
  53         except OSError, e:
  54             pinfo("os.rename(%s, %s): %s" % (fsrc, fdst, e))
  55             raise
  56
  57 class Pkg(BasePkg):
  58     def __init__(self, nvr, tree):
  59         BasePkg.__init__(self, nvr, tree)
  60         self.name=string.join(nvr.split('-')[:-2], '-')
  61         self.version=nvr.split('-')[-2]
  62         self.release=nvr.split('-')[-1]
  63         self.marked4removal=False
  64         self.marked4moving=False
  65         self.marked4movingpool=[]
  66         self.errors=[]
  67         self.warnings=[]
  68
  69     def __cmp__(self, pkg):
  70         if self.name > pkg.name:
  71             return 1
  72         elif self.name < pkg.name:
  73             return -1
  74         else:
  75             return rpm.labelCompare(('0', self.version, self.release),
  76                                     ('0', pkg.version, pkg.release))
  77
  78     def mark4moving(self):
  79         if not self.marked4moving:
  80             # Only one pkg in this pool can be marked for moving
  81             for pkg in self.marked4movingpool:
  82                 pkg.unmark4moving()
  83             self.tree.marked4moving.append(self)
  84             self.marked4moving=True
  85
  86     def unmark4moving(self):
  87         if self.marked4moving:
  88             self.tree.marked4moving.remove(self)
  89             self.marked4moving=False
  90
  91     def mark4removal(self):
  92         if not self.marked4removal:
  93             self.tree.marked4removal.append(self)
  94             self.marked4removal=True
  95
  96     def error(self, msg):
  97         self.errors.append(msg)
  98         if not quietmode:
  99             perror('%s %s' % (self.nvr, msg))
 100
 101     def warning(self, msg):
 102         self.warnings.append(msg)
 103         if not quietmode:
 104             pwarning('%s %s' % (self.nvr, msg))
 105
 106     def load(self, content=None):
 107         BasePkg.load(self, content)
 108         if self.info.has_key('move'):
 109             self.mark4moving()
 110
 111     def writeinfo(self):
 112         f=open(self.tree.basedir+'/SRPMS/.metadata/'+self.nvr+'.src.rpm.info', 'w')
 113         for bid in self.build.keys():
 114             f.write("info:build:%s:requester:%s\ninfo:build:%s:requester_email:%s\n" % (bid, self.build[bid].requester, bid, self.build[bid].requester_email))
 115         for key in self.info.keys():
 116             f.write("info:%s:%s\n" % (key, string.join(self.info[key], ':')))
 117         for arch in self.files.keys():
 118             for rpm in self.files[arch]:
 119                 f.write("file:%s:%s\n" % (arch, rpm))
 120
 121     def remove(self, test=False):
 122         for arch in self.files.keys():
 123             for rpm in self.files[arch]:
 124                 rm(self.tree.basedir+'/'+arch+'/RPMS/'+rpm, test)
 125                 if arch=='noarch':
 126                     if fileexists(noarchcachedir+rpm+'.filelist'):
 127                         rm(noarchcachedir+rpm+'.filelist', test)
 128                     if fileexists(noarchcachedir+rpm+'.reqlist'):
 129                         rm(noarchcachedir+rpm+'.reqlist', test)
 130         rm(self.tree.basedir+'/SRPMS/.metadata/'+self.nvr+'.src.rpm.info', test)
 131
 132     def move(self, dsttree, test=False):
 133         if dsttree.has_key(self.nvr):
 134             movedany=False
 135             for arch in self.files.keys():
 136                 if arch in dsttree[self.nvr].files.keys():
 137                     msg = ""
 138                     if test:
 139                         msg = "TEST "
 140                     pinfo("%sArch %s for %s is already present in dest tree; removing from srctree" % (msg, arch, self.nvr))
 141                     for rpm in self.files[arch]:
 142                         rm(self.tree.basedir+'/'+arch+'/RPMS/'+rpm, test)
 143                 else:
 144                     movedany=True
 145                     dsttree[self.nvr].files[arch]=self.files[arch]
 146                     for rpm in self.files[arch]:
 147                         mv(self.tree.basedir+'/'+arch+'/RPMS/'+rpm, dsttree.basedir+'/'+arch+'/RPMS/', test)
 148             if not test and  movedany:
 149                 for bid in self.build.keys():
 150                     dsttree[self.nvr].build[bid]=self.build[bid]
 151                 dsttree[self.nvr].writeinfo()
 152             rm(self.tree.basedir+'/SRPMS/.metadata/'+self.nvr+'.src.rpm.info', test)
 153         else:
 154             for arch in self.files.keys():
 155                 for rpm in self.files[arch]:
 156                     mv(self.tree.basedir+'/'+arch+'/RPMS/'+rpm, dsttree.basedir+'/'+arch+'/RPMS/', test)
 157             mv(self.tree.basedir+'/SRPMS/.metadata/'+self.nvr+'.src.rpm.info', dsttree.basedir+'/SRPMS/.metadata/', test)
 158
 159
 160 class FtpTree(BaseFtpTree):
 161     def __init__(self, tree, loadall=False):
 162         BaseFtpTree.__init__(self, tree)
 163         self.loadedpkgs={}
 164         self.marked4removal=[]
 165         self.marked4moving=[]
 166         self.pkgnames=[]
 167         self.__loadpkgnames()
 168         if loadall:
 169             for pkgname in self.pkgnames:
 170                 self.loadedpkgs[pkgname]=Pkg(pkgname, self)
 171         # Tests:
 172         self.do_checkbuild=True
 173
 174     def __getitem__(self, key):
 175         if self.loadedpkgs.has_key(key):
 176             return self.loadedpkgs[key]
 177         elif key in self.pkgnames:
 178             pkg=Pkg(key, self)
 179             self.loadedpkgs[key]=pkg
 180             return pkg
 181         else:
 182             raise KeyError, key
 183
 184     def has_key(self, key):
 185         if key in self.pkgnames:
 186             return True
 187         else:
 188             return False
 189
 190     def keys(self):
 191         return self.pkgnames
 192
 193     def values(self):
 194         return self.loadedpkgs.values()
 195
 196     def checktree(self, dsttree):
 197         self.__checkbuild(self.loadedpkgs.values())
 198         self.__checkarchs(dsttree, self.loadedpkgs.values())
 199
 200     def testmove(self, dsttree):
 201         self.__checkbuild(self.marked4moving)
 202         self.__checkarchs(dsttree, self.marked4moving)
 203
 204         self.__rmolderfromsrc(test=True)
 205         self.__rmotherfromdst(dsttree, test=True)
 206
 207         for pkg in self.marked4moving:
 208             pkg.move(dsttree, test=True)
 209
 210     def movepkgs(self, dsttree):
 211         if self.do_checkbuild:
 212             self.__checkbuild(self.marked4moving)
 213         bailoutonerror()
 214         self.__checkarchs(dsttree, self.marked4moving)
 215         bailoutonerror()
 216         self.__rmolderfromsrc()
 217         self.__rmotherfromdst(dsttree)
 218
 219         for pkg in self.marked4moving:
 220             pkg.move(dsttree)
 221
 222     def removepkgs(self):
 223         if self.do_checkbuild:
 224             self.__checkbuild(self.marked4removal)
 225         bailoutonerror()
 226         for pkg in self.marked4removal:
 227             pkg.remove()
 228
 229     def mark4removal(self, wannabepkgs):
 230         self.__mark4something(wannabepkgs, Pkg.mark4removal)
 231
 232     def mark4moving(self, wannabepkgs):
 233         self.__mark4something(wannabepkgs, Pkg.mark4moving)
 234
 235
 236     # Internal functions below
 237
 238     def __loadpkgnames(self):
 239         def checkfiletype(name):
 240             if name[-13:]=='.src.rpm.info':
 241                 return True
 242             else:
 243                 return False
 244         list=filter(checkfiletype, os.listdir(self.basedir+'/SRPMS/.metadata'))
 245         self.pkgnames=map((lambda x: x[:-13]), list)
 246
 247     def __mark4something(self, wannabepkgs, markfunction):
 248         def chopoffextension(pkg):
 249             found=pkg.find('.src.rpm')
 250             if found==-1:
 251                 return pkg
 252             else:
 253                 return pkg[:found]
 254         for wannabepkg in wannabepkgs:
 255             pkgname=chopoffextension(wannabepkg)
 256             if pkgname in self.pkgnames:
 257                 if not pkgname in self.loadedpkgs.keys():
 258                     self.loadedpkgs[pkgname]=Pkg(pkgname, self)
 259                 markfunction(self.loadedpkgs[pkgname])
 260             else:
 261                 perror('%s not found in source tree' % pkgname)
 262         bailoutonerror()
 263
 264     def __checkbuild(self, marked):
 265         f=urllib.urlopen(config.builderqueue)
 266         #f=open('queue.txt')
 267         requests={}
 268         reid=re.compile(r'^.*id=(.*) pri.*$')
 269         regb=re.compile(r'^group:.*$|builders:.*$', re.M)
 270         for i in re.findall(regb, f.read()):
 271             if i[0]=='g':
 272                 id=reid.sub(r'\1', i)
 273                 requests[id]=""
 274             elif i[0]=='b':
 275                 requests[id]=requests[id]+i
 276         f.close()
 277         for pkg in marked:
 278             for bid in pkg.build.keys():
 279                 if requests.has_key(bid) and not requests[bid].find('?') == -1:
 280                     pkg.error("(buildid %s) building not finished" % bid)
 281
 282     def __checkarchs(self, dsttree, marked):
 283         for pkg in marked:
 284             if len(pkg.files.keys()) <= 1:
 285                 pkg.error('has only src.rpm built')
 286                 continue
 287             otherpkgnames=self.__find_other_pkgs(pkg, dsttree)
 288             if otherpkgnames: # check if we're not removing some archs
 289                 curarchs=[]
 290                 missingarchs=[]
 291                 for somepkg in otherpkgnames:
 292                     curarchs.extend(Pkg(somepkg, dsttree).files.keys())
 293                 for arch in curarchs:
 294                     if arch not in pkg.files.keys():
 295                         missingarchs.append(arch)
 296                 if missingarchs:
 297                     pkg.error('moving would remove archs: %s' % missingarchs)
 298             else: # warn if a package isn't built for all archs
 299                 if (config.separate_noarch and 'noarch' in pkg.files.keys() and
 300                                     len(pkg.files.keys())==2):
 301                     continue
 302                 elif len(pkg.files.keys()) != len(config.ftp_archs)+1:
 303                     missingarchs=[]
 304                     for arch in config.ftp_archs:
 305                         if arch not in pkg.files.keys():
 306                             missingarchs.append(arch)
 307                     pkg.warning('not built for archs: %s' % missingarchs)
 308
 309     def __rmolderfromsrc(self, test=False):
 310         for pkg in self.marked4moving:
 311             olderpkgnames=self.__find_older_pkgs(pkg)
 312             for i in olderpkgnames:
 313                 Pkg(i, self).remove(test)
 314
 315     def __rmotherfromdst(self, dsttree, test=False):
 316         for pkg in self.marked4moving:
 317             pkgnames=self.__find_other_pkgs(pkg, dsttree)
 318             for i in pkgnames:
 319                 Pkg(i, dsttree).remove(test)
 320
 321     # Used more than once filter functions
 322
 323     def __find_other_pkgs(self, pkg, tree):
 324         escapedpkgname=pkg.name.replace('.', '\.').replace('+', '\+')
 325         ziewre=re.compile(escapedpkgname+'-[^-]*-[^-]*$')
 326         def filter_other_pkgs(x):
 327             if ziewre.match(x) and not x == pkg.nvr:
 328                 return True
 329             else:
 330                 return False
 331         return filter(filter_other_pkgs, tree.pkgnames)
 332
 333     def __find_older_pkgs(self, pkg):
 334         def filter_older_pkgs(x):
 335             c=x.split('-')
 336             rc = rpm.labelCompare(('0', pkg.version, pkg.release),
 337                                                         ('0', c[-2], c[-1]))
 338             if rc == 1: # pkg > x
 339                 return True
 340             else:
 341                 return False
 342         return filter(filter_older_pkgs, self.__find_other_pkgs(pkg, self))
 343