modules/ftptree.py

   1 # vi: encoding=utf-8 ts=8 sts=4 sw=4 et
   2
   3 import os, config, string, urllib, re, rpm
   4 from common import fileexists, noarchcachedir
   5 from baseftptree import BasePkg, BaseFtpTree
   6 from sign import is_signed
   7
   8 errnum = 0
   9 quietmode = False
  10
  11 class SomeError(Exception):
  12     def __init__(self):
  13         return
  14
  15     def __str__(self):
  16         print "","An Error occured!"
  17
  18 def bailoutonerror():
  19     if not errnum == 0:
  20         print "%d error(s) encountered... aborting" % errnum
  21         raise SomeError
  22
  23 def pinfo(msg):
  24     print 'INFO: ' + msg
  25
  26 def perror(msg):
  27     global errnum
  28     errnum = errnum + 1
  29     print 'ERR: ' + msg
  30
  31 def pwarning(msg):
  32     print 'WARN: ' + msg
  33
  34 def rm(file, test = False):
  35     if test:
  36         if not os.path.exists(file):
  37             pinfo("TEST os.remove(%s): file doesn't exists" % file)
  38     else:
  39         try:
  40             os.remove(file)
  41         except OSError, e:
  42             pinfo("os.remove(%s): %s" % (file, e))
  43             #raise
  44
  45 def mv(src, dst, test = False):
  46     fsrc = src
  47     fdst = dst + '/' + src.split('/')[-1]
  48     if test:
  49         if not os.path.exists(src):
  50             pinfo("TEST os.rename(%s, %s): source doesn't exists" % (fsrc, fdst))
  51         if not os.path.exists(dst):
  52             pinfo("TEST destination doesn't exist: %s" % dst)
  53     else:
  54         try:
  55             os.rename(fsrc, fdst)
  56         except OSError, e:
  57             pinfo("os.rename(%s, %s): %s" % (fsrc, fdst, e))
  58             raise
  59
  60 class Pkg(BasePkg):
  61     def __init__(self, nvr, tree):
  62         BasePkg.__init__(self, nvr, tree)
  63         self.name = string.join(nvr.split('-')[:-2], '-')
  64         self.version = nvr.split('-')[-2]
  65         self.release = nvr.split('-')[-1]
  66         self.marked4removal = False
  67         self.marked4moving = False
  68         self.marked4movingpool = []
  69         self.errors = []
  70         self.warnings = []
  71
  72     def __cmp__(self, pkg):
  73         if self.name > pkg.name:
  74             return 1
  75         elif self.name < pkg.name:
  76             return -1
  77         else:
  78             return rpm.labelCompare(('0', self.version, self.release),
  79                                     ('0', pkg.version, pkg.release))
  80
  81
  82     # unfortunately can't do new Pkg(NVR), and have no "tree" in this pkg context
  83     # so this static function
  84     def is_debuginfo(self, nvr):
  85         """
  86         returns true if NVR is debuginfo package and separate debuginfo is enabled
  87         """
  88         if not config.separate_debuginfo:
  89             return False
  90         pkg = nvr.split('-')[:-2]
  91         return pkg[-1] == 'debuginfo'
  92
  93     def mark4moving(self):
  94         if not self.marked4moving:
  95             # Only one pkg in this pool can be marked for moving
  96             for pkg in self.marked4movingpool:
  97                 pkg.unmark4moving()
  98             self.tree.marked4moving.append(self)
  99             self.marked4moving=True
 100
 101     def unmark4moving(self):
 102         if self.marked4moving:
 103             self.tree.marked4moving.remove(self)
 104             self.marked4moving=False
 105
 106     def mark4removal(self):
 107         if not self.marked4removal:
 108             self.tree.marked4removal.append(self)
 109             self.marked4removal=True
 110
 111     def error(self, msg):
 112         self.errors.append(msg)
 113         if not quietmode:
 114             perror('%s %s' % (self.nvr, msg))
 115
 116     def warning(self, msg):
 117         self.warnings.append(msg)
 118         if not quietmode:
 119             pwarning('%s %s' % (self.nvr, msg))
 120
 121     def load(self, content=None):
 122         BasePkg.load(self, content)
 123         if self.info.has_key('move'):
 124             self.mark4moving()
 125
 126     def writeinfo(self):
 127         f = open(self.tree.basedir+'/SRPMS/.metadata/'+self.nvr+'.src.rpm.info', 'w')
 128         for bid in self.build.keys():
 129             f.write("info:build:%s:requester:%s\ninfo:build:%s:requester_email:%s\n" % (bid, self.build[bid].requester, bid, self.build[bid].requester_email))
 130         for key in self.info.keys():
 131             f.write("info:%s:%s\n" % (key, string.join(self.info[key], ':')))
 132         for arch in self.files.keys():
 133             for rpm in self.files[arch]:
 134                 f.write("file:%s:%s\n" % (arch, rpm))
 135
 136     def remove(self, test = False):
 137         """
 138         Remove package from ftp
 139         """
 140         for arch in self.files.keys():
 141             for rpm in self.files[arch]:
 142                 if self.is_debuginfo(rpm):
 143                     rm(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, test)
 144                 else:
 145                     rm(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, test)
 146                 if arch == 'noarch':
 147                     if fileexists(noarchcachedir + rpm + '.filelist'):
 148                         rm(noarchcachedir + rpm + '.filelist', test)
 149                     if fileexists(noarchcachedir + rpm + '.reqlist'):
 150                         rm(noarchcachedir + rpm + '.reqlist', test)
 151         rm(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', test)
 152
 153     def rpmfiles(self):
 154         """
 155         Return rpm files related to this package
 156         """
 157         files = []
 158         for arch, rpms in self.files.items():
 159             for nvr in rpms:
 160                 if self.is_debuginfo(nvr):
 161                     files.append(self.tree.basedir + '/' + arch + '/debuginfo/' + nvr)
 162                 else:
 163                     files.append(self.tree.basedir + '/' + arch + '/RPMS/' + nvr)
 164         return files
 165
 166     def obsoletes(self):
 167         """
 168         Return obsoletes for all packages in Pkg:
 169
 170         {'php-geshi': set(['geshi'])}
 171
 172         """
 173         def rpmhdr(pkg):
 174             ts = rpm.ts()
 175             ts.setVSFlags(rpm._RPMVSF_NOSIGNATURES)
 176             fdno = os.open(pkg, os.O_RDONLY)
 177             hdr = ts.hdrFromFdno(fdno)
 178             os.close(fdno)
 179             return hdr
 180
 181         obsoletes = {}
 182         for rpmfile in self.rpmfiles():
 183             if not os.path.exists(rpmfile):
 184                 continue
 185             hdr = rpmhdr(rpmfile)
 186             if not hdr[rpm.RPMTAG_OBSOLETES]:
 187                 continue
 188
 189             name = hdr[rpm.RPMTAG_NAME]
 190             if not name in obsoletes:
 191                 obsoletes[name] = set()
 192
 193             for tag in hdr[rpm.RPMTAG_OBSOLETES]:
 194                 obsoletes[name].add(tag)
 195
 196         return obsoletes
 197
 198     def move(self, dsttree, test = False):
 199         if dsttree.has_key(self.nvr):
 200             movedany = False
 201             for arch in self.files.keys():
 202                 if arch in dsttree[self.nvr].files.keys():
 203                     msg = ""
 204                     if test:
 205                         msg = "TEST "
 206                     pinfo("%sArch %s for %s is already present in dest tree; removing from srctree" % (msg, arch, self.nvr))
 207                     for rpm in self.files[arch]:
 208                         if self.is_debuginfo(rpm):
 209                             rm(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, test)
 210                         else:
 211                             rm(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, test)
 212                 else:
 213                     movedany = True
 214                     dsttree[self.nvr].files[arch] = self.files[arch]
 215                     for rpm in self.files[arch]:
 216                         if self.is_debuginfo(rpm):
 217                             mv(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, dsttree.basedir + '/' + arch + '/debuginfo/', test)
 218                         else:
 219                             mv(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, dsttree.basedir + '/' + arch + '/RPMS/', test)
 220             if not test and movedany:
 221                 for bid in self.build.keys():
 222                     dsttree[self.nvr].build[bid] = self.build[bid]
 223                 dsttree[self.nvr].writeinfo()
 224             rm(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', test)
 225         else:
 226             # move files
 227             for arch in self.files.keys():
 228                 for rpm in self.files[arch]:
 229                     if self.is_debuginfo(rpm):
 230                         mv(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, dsttree.basedir + '/' + arch + '/debuginfo/', test)
 231                     else:
 232                         mv(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, dsttree.basedir + '/' + arch + '/RPMS/', test)
 233
 234             # move metadata
 235             mv(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', dsttree.basedir + '/SRPMS/.metadata/', test)
 236
 237 class FtpTree(BaseFtpTree):
 238     def __init__(self, tree, loadall=False):
 239         BaseFtpTree.__init__(self, tree)
 240         self.loadedpkgs = {}
 241         self.marked4removal = []
 242         self.marked4moving = []
 243         self.pkgnames = []
 244         self.__loadpkgnames()
 245         if loadall:
 246             for pkgname in self.pkgnames:
 247                 self.loadedpkgs[pkgname] = Pkg(pkgname, self)
 248         # Tests:
 249         self.do_checkbuild = True
 250
 251     def __getitem__(self, key):
 252         if self.loadedpkgs.has_key(key):
 253             return self.loadedpkgs[key]
 254         elif key in self.pkgnames:
 255             pkg=Pkg(key, self)
 256             self.loadedpkgs[key]=pkg
 257             return pkg
 258         else:
 259             raise KeyError, key
 260
 261     def has_key(self, key):
 262         if key in self.pkgnames:
 263             return True
 264         else:
 265             return False
 266
 267     def keys(self):
 268         return self.pkgnames
 269
 270     def values(self):
 271         return self.loadedpkgs.values()
 272
 273     def checktree(self, dsttree):
 274         self.__checkbuild(self.loadedpkgs.values())
 275         self.__checkarchs(dsttree, self.loadedpkgs.values())
 276
 277     def testmove(self, dsttree, archivetree = None):
 278         self.__checkbuild(self.marked4moving)
 279         self.__checkarchs(dsttree, self.marked4moving)
 280
 281         self.__checksigns(dsttree, self.marked4moving, test = True)
 282         self.__checkforobsoletes(dsttree, self.marked4moving, test = True)
 283
 284         self.__rmolderfromsrc(test = True)
 285         self.__rmotherfromdst(dsttree, test = True, archivetree = archivetree)
 286
 287         for pkg in self.marked4moving:
 288             pkg.move(dsttree, test = True)
 289
 290     def movepkgs(self, dsttree, archivetree = None):
 291         if self.do_checkbuild:
 292             self.__checkbuild(self.marked4moving)
 293         bailoutonerror()
 294
 295         self.__checkarchs(dsttree, self.marked4moving)
 296         bailoutonerror()
 297
 298         self.__checksigns(dsttree, self.marked4moving)
 299         bailoutonerror()
 300
 301         self.__rmolderfromsrc()
 302         self.__rmotherfromdst(dsttree, archivetree = archivetree)
 303
 304         for pkg in self.marked4moving:
 305             pkg.move(dsttree)
 306
 307     def rpmfiles(self):
 308         if self.do_checkbuild:
 309             self.__checkbuild(self.marked4moving)
 310
 311         files = []
 312         for pkg in self.marked4moving:
 313             files += pkg.rpmfiles()
 314         return files
 315
 316     def removepkgs(self):
 317         if self.do_checkbuild:
 318             self.__checkbuild(self.marked4removal)
 319         bailoutonerror()
 320         for pkg in self.marked4removal:
 321             pkg.remove()
 322
 323     def mark4removal(self, wannabepkgs):
 324         self.__mark4something(wannabepkgs, Pkg.mark4removal)
 325
 326     def mark4moving(self, wannabepkgs):
 327         self.__mark4something(wannabepkgs, Pkg.mark4moving)
 328
 329
 330     # Internal functions below
 331     def __arch_stringify(self, list):
 332         ret = []
 333         # XXX: is dist line in any config?
 334         dist = 'ac'
 335         for arch in list:
 336             ret.append(dist + '-' + arch)
 337         return ' '.join(ret)
 338
 339     def __loadpkgnames(self):
 340         def checkfiletype(name):
 341             if name[-13:]=='.src.rpm.info':
 342                 return True
 343             else:
 344                 return False
 345         list = filter(checkfiletype, os.listdir(self.basedir+'/SRPMS/.metadata'))
 346         self.pkgnames = map((lambda x: x[:-13]), list)
 347
 348     def __mark4something(self, wannabepkgs, markfunction):
 349         def chopoffextension(pkg):
 350             found = pkg.find('.src.rpm')
 351             if found == -1:
 352                 return pkg
 353             else:
 354                 return pkg[:found]
 355
 356         for wannabepkg in wannabepkgs:
 357             pkgname = chopoffextension(wannabepkg)
 358             if pkgname in self.pkgnames:
 359                 if not pkgname in self.loadedpkgs.keys():
 360                     self.loadedpkgs[pkgname]=Pkg(pkgname, self)
 361                 markfunction(self.loadedpkgs[pkgname])
 362             else:
 363                 perror('%s not found in source tree' % pkgname)
 364         bailoutonerror()
 365
 366     def __checkbuild(self, marked):
 367         """
 368         Checks queue file if all arches are built
 369
 370         Reads config.builderqueue to grab the info
 371         """
 372         f = urllib.urlopen(config.builderqueue)
 373         requests = {}
 374         reid = re.compile(r'^.*id=(.*) pri.*$')
 375         regb = re.compile(r'^group:.*$|builders:.*$', re.M)
 376         for i in re.findall(regb, f.read()):
 377             if i[0] == 'g':
 378                 id = reid.sub(r'\1', i)
 379                 requests[id] = ""
 380             elif i[0]=='b':
 381                 requests[id] = requests[id] + i
 382         f.close()
 383
 384         for pkg in marked:
 385             for bid in pkg.build.keys():
 386                 if requests.has_key(bid) and not requests[bid].find('?') == -1:
 387                     pkg.error("(buildid %s) building not finished" % bid)
 388
 389     def __checkarchs(self, dsttree, marked):
 390         """
 391         Checks marked pkgs it is built on all archs.
 392         """
 393         for pkg in marked:
 394             if len(pkg.files.keys()) <= 1:
 395                 pkg.error('has only src.rpm built')
 396                 continue
 397             otherpkgnames = self.__find_other_pkgs(pkg, dsttree)
 398
 399             # check if we're not removing some archs
 400             if otherpkgnames:
 401                 curarchs = []
 402                 missingarchs = []
 403                 for somepkg in otherpkgnames:
 404                     curarchs.extend(Pkg(somepkg, dsttree).files.keys())
 405                 for arch in curarchs:
 406                     if arch not in pkg.files.keys():
 407                         missingarchs.append(arch)
 408                 if missingarchs:
 409                     pkg.error('moving would remove archs: %s' % self.__arch_stringify(missingarchs))
 410             else:
 411                 # warn if a package isn't built for all archs
 412                 if (config.separate_noarch and 'noarch' in pkg.files.keys() and len(pkg.files.keys()) == 2):
 413                     continue
 414                 elif len(pkg.files.keys()) != len(config.ftp_archs) + 1:
 415                     missingarchs = []
 416                     for arch in config.ftp_archs:
 417                         if arch not in pkg.files.keys():
 418                             missingarchs.append(arch)
 419                     pkg.warning('not built for archs: %s' % self.__arch_stringify(missingarchs))
 420
 421     def __rmolderfromsrc(self, test = False):
 422         for pkg in self.marked4moving:
 423             olderpkgnames = self.__find_older_pkgs(pkg)
 424             for i in olderpkgnames:
 425                 Pkg(i, self).remove(test)
 426
 427     def __rmotherfromdst(self, dsttree, test = False, archivetree = None):
 428         for pkg in self.marked4moving:
 429             pkgnames = self.__find_other_pkgs(pkg, dsttree)
 430             for i in pkgnames:
 431                 if archivetree == None:
 432                     Pkg(i, dsttree).remove(test)
 433                 else:
 434                     Pkg(i, dsttree).move(archivetree, test = test)
 435
 436     # Used more than once filter functions
 437     def __find_other_pkgs(self, pkg, tree):
 438         escapedpkgname = pkg.name.replace('.', '\.').replace('+', '\+')
 439         ziewre = re.compile(escapedpkgname + '-[^-]*-[^-]*$')
 440         def filter_other_pkgs(x):
 441             if ziewre.match(x) and not x == pkg.nvr:
 442                 return True
 443             else:
 444                 return False
 445         return filter(filter_other_pkgs, tree.pkgnames)
 446
 447     def __find_older_pkgs(self, pkg):
 448         def filter_older_pkgs(x):
 449             c = x.split('-')
 450             rc = rpm.labelCompare(('0', pkg.version, pkg.release),
 451                                                         ('0', c[-2], c[-1]))
 452             if rc == 1: # pkg > x
 453                 return True
 454             else:
 455                 return False
 456         return filter(filter_older_pkgs, self.__find_other_pkgs(pkg, self))
 457
 458     def __checksigns(self, tree, pkgs, test = False):
 459         """
 460         Checks if pkgs in tree are all signed.
 461
 462         in case of test = true, error flag is set for unsigned packages
 463         """
 464         if not tree.treename in config.signed_trees:
 465             return
 466
 467         for pkg in pkgs:
 468             unsigned = 0
 469             for file in pkg.rpmfiles():
 470                 if not is_signed(file):
 471                     unsigned += 1
 472
 473             if unsigned != 0:
 474                 if test == True:
 475                     if not quietmode:
 476                         pkg.warning('%d files not signed' % unsigned)
 477                 else:
 478                     pkg.error('%d files not signed' % unsigned)
 479
 480     def __checkforobsoletes(self, tree, pkgs, test = False):
 481         """
 482         Checks queue file if package obsoletes something in destination tree and suggest for removal.
 483
 484         Only NAME tag is compared, i.e virtual packages do not get reported.
 485
 486         """
 487         if test != True:
 488             return
 489
 490         def findbyname(name):
 491             def x(nvr):
 492                 return '-'.join(nvr.split('-')[:-2]) == name
 493             return filter(x, tree.pkgnames)
 494
 495         for pkg in pkgs:
 496             obsoletes = pkg.obsoletes()
 497             if not obsoletes:
 498                 continue
 499
 500             for pn, setlist in obsoletes.items():
 501                 for item in setlist:
 502                     p = findbyname(item)
 503                     if p:
 504                         pkg.warning('obsoletes %s (via %s) in dest tree, perhaps you want rmpkg' % (p,pn))