modules/ftptree.py

   1 # vi: encoding=utf-8 ts=8 sts=4 sw=4 et
   2
   3 import os, config, string, urllib, re, rpm
   4 from common import fileexists, noarchcachedir
   5 from baseftptree import BasePkg, BaseFtpTree
   6 from sign import is_signed
   7
   8 errnum = 0
   9 quietmode = False
  10
  11 class SomeError(Exception):
  12     def __init__(self):
  13         return
  14
  15     def __str__(self):
  16         print "","An Error occured!"
  17
  18 def bailoutonerror():
  19     if not errnum == 0:
  20         print "%d error(s) encountered... aborting" % errnum
  21         raise SomeError
  22
  23 def pinfo(msg):
  24     print 'INFO: ' + msg
  25
  26 def perror(msg):
  27     global errnum
  28     errnum = errnum + 1
  29     print 'ERR: ' + msg
  30
  31 def pwarning(msg):
  32     print 'WARN: ' + msg
  33
  34 def rm(file, test = False):
  35     if test:
  36         if not os.path.exists(file):
  37             pinfo("TEST os.remove(%s): file doesn't exists" % file)
  38     else:
  39         try:
  40             os.remove(file)
  41         except OSError, e:
  42             pinfo("os.remove(%s): %s" % (file, e))
  43             #raise
  44
  45 def mv(src, dst, test = False):
  46     fsrc = src
  47     fdst = dst + '/' + src.split('/')[-1]
  48     if test:
  49         if not os.path.exists(src):
  50             pinfo("TEST os.rename(%s, %s): source doesn't exists" % (fsrc, fdst))
  51         if not os.path.exists(dst):
  52             pinfo("TEST destination doesn't exist: %s" % dst)
  53     else:
  54         try:
  55             os.rename(fsrc, fdst)
  56         except OSError, e:
  57             pinfo("os.rename(%s, %s): %s" % (fsrc, fdst, e))
  58             raise
  59
  60 class Pkg(BasePkg):
  61     def __init__(self, nvr, tree):
  62         BasePkg.__init__(self, nvr, tree)
  63         self.name = string.join(nvr.split('-')[:-2], '-')
  64         self.version = nvr.split('-')[-2]
  65         self.release = nvr.split('-')[-1]
  66         self.marked4removal = False
  67         self.marked4moving = False
  68         self.marked4movingpool = []
  69         self.errors = []
  70         self.warnings = []
  71
  72     def __cmp__(self, pkg):
  73         if self.name > pkg.name:
  74             return 1
  75         elif self.name < pkg.name:
  76             return -1
  77         else:
  78             return rpm.labelCompare(('0', self.version, self.release),
  79                                     ('0', pkg.version, pkg.release))
  80
  81
  82     # unfortunately can't do new Pkg(NVR), and have no "tree" in this pkg context
  83     # so this static function
  84     def is_debuginfo(self, nvr):
  85         """
  86         returns true if NVR is debuginfo package and separate debuginfo is enabled
  87         """
  88         if not config.separate_debuginfo:
  89             return False
  90         pkg = nvr.split('-')[:-2]
  91         return pkg[-1] == 'debuginfo'
  92
  93     # returns true if package build is integer
  94     def is_release(self):
  95         """
  96         To account Release tags with subver macros, we consider integer release
  97         if it contains odd number of dots:
  98
  99         1 -> True
 100         0.1 -> False
 101         0.%{subver}.%{rel}, %{rel} = 1 -> 0.20010.1 -> True
 102         0.%{subver}.%{rel}, %{rel} = 0.1 -> 0.20010.0.1 -> False
 103         """
 104         # count elements by splitting by dot
 105         parts = self.release.split('.')
 106         return len(parts) % 2 == 1
 107
 108     def mark4moving(self):
 109         if not self.marked4moving:
 110             # Only one pkg in this pool can be marked for moving
 111             for pkg in self.marked4movingpool:
 112                 pkg.unmark4moving()
 113             self.tree.marked4moving.append(self)
 114             self.marked4moving=True
 115
 116     def unmark4moving(self):
 117         if self.marked4moving:
 118             self.tree.marked4moving.remove(self)
 119             self.marked4moving=False
 120
 121     def mark4removal(self):
 122         if not self.marked4removal:
 123             self.tree.marked4removal.append(self)
 124             self.marked4removal=True
 125
 126     def error(self, msg):
 127         self.errors.append(msg)
 128         if not quietmode:
 129             perror('%s %s' % (self.nvr, msg))
 130
 131     def warning(self, msg):
 132         self.warnings.append(msg)
 133         if not quietmode:
 134             pwarning('%s %s' % (self.nvr, msg))
 135
 136     def load(self, content=None):
 137         BasePkg.load(self, content)
 138         if self.info.has_key('move'):
 139             self.mark4moving()
 140
 141     def writeinfo(self):
 142         f = open(self.tree.basedir+'/SRPMS/.metadata/'+self.nvr+'.src.rpm.info', 'w')
 143         for bid in self.build.keys():
 144             f.write("info:build:%s:requester:%s\ninfo:build:%s:requester_email:%s\n" % (bid, self.build[bid].requester, bid, self.build[bid].requester_email))
 145         for key in self.info.keys():
 146             f.write("info:%s:%s\n" % (key, string.join(self.info[key], ':')))
 147         for arch in self.files.keys():
 148             for rpm in self.files[arch]:
 149                 f.write("file:%s:%s\n" % (arch, rpm))
 150
 151     def remove(self, test = False):
 152         """
 153         Remove package from ftp
 154         """
 155         for arch in self.files.keys():
 156             for rpm in self.files[arch]:
 157                 if self.is_debuginfo(rpm):
 158                     rm(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, test)
 159                 else:
 160                     rm(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, test)
 161                 if arch == 'noarch':
 162                     if fileexists(noarchcachedir + rpm + '.filelist'):
 163                         rm(noarchcachedir + rpm + '.filelist', test)
 164                     if fileexists(noarchcachedir + rpm + '.reqlist'):
 165                         rm(noarchcachedir + rpm + '.reqlist', test)
 166         rm(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', test)
 167
 168     def rpmfiles(self):
 169         """
 170         Return rpm files related to this package
 171         """
 172         files = []
 173         for arch, rpms in self.files.items():
 174             for nvr in rpms:
 175                 if self.is_debuginfo(nvr):
 176                     files.append(self.tree.basedir + '/' + arch + '/debuginfo/' + nvr)
 177                 else:
 178                     files.append(self.tree.basedir + '/' + arch + '/RPMS/' + nvr)
 179         return files
 180
 181     def obsoletes(self):
 182         """
 183         Return obsoletes for all packages in Pkg:
 184
 185         {'php-geshi': set(['geshi'])}
 186
 187         """
 188         def rpmhdr(pkg):
 189             ts = rpm.ts()
 190             ts.setVSFlags(rpm._RPMVSF_NOSIGNATURES)
 191             fdno = os.open(pkg, os.O_RDONLY)
 192             hdr = ts.hdrFromFdno(fdno)
 193             os.close(fdno)
 194             return hdr
 195
 196         obsoletes = {}
 197         for rpmfile in self.rpmfiles():
 198             if not os.path.exists(rpmfile):
 199                 continue
 200             hdr = rpmhdr(rpmfile)
 201             if not hdr[rpm.RPMTAG_OBSOLETES]:
 202                 continue
 203
 204             name = hdr[rpm.RPMTAG_NAME]
 205             if not name in obsoletes:
 206                 obsoletes[name] = set()
 207
 208             for tag in hdr[rpm.RPMTAG_OBSOLETES]:
 209                 obsoletes[name].add(tag)
 210
 211         return obsoletes
 212
 213     def move(self, dsttree, test = False):
 214         if dsttree.has_key(self.nvr):
 215             movedany = False
 216             for arch in self.files.keys():
 217                 if arch in dsttree[self.nvr].files.keys():
 218                     msg = ""
 219                     if test:
 220                         msg = "TEST "
 221                     pinfo("%sArch %s for %s is already present in dest tree; removing from srctree" % (msg, arch, self.nvr))
 222                     for rpm in self.files[arch]:
 223                         if self.is_debuginfo(rpm):
 224                             rm(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, test)
 225                         else:
 226                             rm(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, test)
 227                 else:
 228                     movedany = True
 229                     dsttree[self.nvr].files[arch] = self.files[arch]
 230                     for rpm in self.files[arch]:
 231                         if self.is_debuginfo(rpm):
 232                             mv(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, dsttree.basedir + '/' + arch + '/debuginfo/', test)
 233                         else:
 234                             mv(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, dsttree.basedir + '/' + arch + '/RPMS/', test)
 235             if not test and movedany:
 236                 for bid in self.build.keys():
 237                     dsttree[self.nvr].build[bid] = self.build[bid]
 238                 dsttree[self.nvr].writeinfo()
 239             rm(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', test)
 240         else:
 241             # move files
 242             for arch in self.files.keys():
 243                 for rpm in self.files[arch]:
 244                     if self.is_debuginfo(rpm):
 245                         mv(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, dsttree.basedir + '/' + arch + '/debuginfo/', test)
 246                     else:
 247                         mv(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, dsttree.basedir + '/' + arch + '/RPMS/', test)
 248
 249             # move metadata
 250             mv(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', dsttree.basedir + '/SRPMS/.metadata/', test)
 251
 252 class FtpTree(BaseFtpTree):
 253     def __init__(self, tree, loadall=False):
 254         BaseFtpTree.__init__(self, tree)
 255         self.loadedpkgs = {}
 256         self.marked4removal = []
 257         self.marked4moving = []
 258         self.pkgnames = []
 259         self.__loadpkgnames()
 260         if loadall:
 261             for pkgname in self.pkgnames:
 262                 self.loadedpkgs[pkgname] = Pkg(pkgname, self)
 263         # Tests:
 264         self.do_checkbuild = True
 265
 266     def __getitem__(self, key):
 267         if self.loadedpkgs.has_key(key):
 268             return self.loadedpkgs[key]
 269         elif key in self.pkgnames:
 270             pkg=Pkg(key, self)
 271             self.loadedpkgs[key]=pkg
 272             return pkg
 273         else:
 274             raise KeyError, key
 275
 276     def has_key(self, key):
 277         if key in self.pkgnames:
 278             return True
 279         else:
 280             return False
 281
 282     def keys(self):
 283         return self.pkgnames
 284
 285     def values(self):
 286         return self.loadedpkgs.values()
 287
 288     def checktree(self, dsttree):
 289         self.__checkbuild(self.loadedpkgs.values())
 290         self.__checkarchs(dsttree, self.loadedpkgs.values())
 291
 292     def testmove(self, dsttree, archivetree = None):
 293         self.__checkbuild(self.marked4moving)
 294         self.__checkarchs(dsttree, self.marked4moving)
 295
 296         self.__checksigns(dsttree, self.marked4moving, test = True)
 297         self.__checkforobsoletes(dsttree, self.marked4moving, test = True)
 298         self.__checkforrelease(dsttree, self.marked4moving, test = True)
 299
 300         self.__rmolderfromsrc(test = True)
 301         self.__rmotherfromdst(dsttree, test = True, archivetree = archivetree)
 302
 303         for pkg in self.marked4moving:
 304             pkg.move(dsttree, test = True)
 305
 306     def movepkgs(self, dsttree, archivetree = None):
 307         if self.do_checkbuild:
 308             self.__checkbuild(self.marked4moving)
 309         bailoutonerror()
 310
 311         self.__checkarchs(dsttree, self.marked4moving)
 312         bailoutonerror()
 313
 314         self.__checksigns(dsttree, self.marked4moving)
 315         bailoutonerror()
 316
 317         self.__rmolderfromsrc()
 318         self.__rmotherfromdst(dsttree, archivetree = archivetree)
 319
 320         for pkg in self.marked4moving:
 321             pkg.move(dsttree)
 322
 323     def rpmfiles(self):
 324         if self.do_checkbuild:
 325             self.__checkbuild(self.marked4moving)
 326
 327         files = []
 328         for pkg in self.marked4moving:
 329             files += pkg.rpmfiles()
 330         return files
 331
 332     def removepkgs(self):
 333         if self.do_checkbuild:
 334             self.__checkbuild(self.marked4removal)
 335         bailoutonerror()
 336         for pkg in self.marked4removal:
 337             pkg.remove()
 338
 339     def mark4removal(self, wannabepkgs):
 340         self.__mark4something(wannabepkgs, Pkg.mark4removal)
 341
 342     def mark4moving(self, wannabepkgs):
 343         self.__mark4something(wannabepkgs, Pkg.mark4moving)
 344
 345
 346     # Internal functions below
 347     def __arch_stringify(self, list):
 348         ret = []
 349         # XXX: is dist line in any config?
 350         dist = 'ac'
 351         for arch in list:
 352             ret.append(dist + '-' + arch)
 353         return ' '.join(ret)
 354
 355     def __loadpkgnames(self):
 356         def checkfiletype(name):
 357             if name[-13:]=='.src.rpm.info':
 358                 return True
 359             else:
 360                 return False
 361         list = filter(checkfiletype, os.listdir(self.basedir+'/SRPMS/.metadata'))
 362         self.pkgnames = map((lambda x: x[:-13]), list)
 363
 364     def __mark4something(self, wannabepkgs, markfunction):
 365         def chopoffextension(pkg):
 366             found = pkg.find('.src.rpm')
 367             if found == -1:
 368                 return pkg
 369             else:
 370                 return pkg[:found]
 371
 372         for wannabepkg in wannabepkgs:
 373             pkgname = chopoffextension(wannabepkg)
 374             if pkgname in self.pkgnames:
 375                 if not pkgname in self.loadedpkgs.keys():
 376                     self.loadedpkgs[pkgname]=Pkg(pkgname, self)
 377                 markfunction(self.loadedpkgs[pkgname])
 378             else:
 379                 perror('%s not found in source tree' % pkgname)
 380         bailoutonerror()
 381
 382     def __checkbuild(self, marked):
 383         """
 384         Checks queue file if all arches are built
 385
 386         Reads config.builderqueue to grab the info
 387         """
 388         f = urllib.urlopen(config.builderqueue)
 389         requests = {}
 390         reid = re.compile(r'^.*id=(.*) pri.*$')
 391         regb = re.compile(r'^group:.*$|builders:.*$', re.M)
 392         for i in re.findall(regb, f.read()):
 393             if i[0] == 'g':
 394                 id = reid.sub(r'\1', i)
 395                 requests[id] = ""
 396             elif i[0]=='b':
 397                 requests[id] = requests[id] + i
 398         f.close()
 399
 400         for pkg in marked:
 401             for bid in pkg.build.keys():
 402                 if requests.has_key(bid) and not requests[bid].find('?') == -1:
 403                     pkg.error("(buildid %s) building not finished" % bid)
 404
 405     def __checkarchs(self, dsttree, marked):
 406         """
 407         Checks marked pkgs it is built on all archs.
 408         """
 409         for pkg in marked:
 410             if len(pkg.files.keys()) <= 1:
 411                 pkg.error('has only src.rpm built')
 412                 continue
 413             otherpkgnames = self.__find_other_pkgs(pkg, dsttree)
 414
 415             # check if we're not removing some archs
 416             if otherpkgnames:
 417                 curarchs = []
 418                 missingarchs = []
 419                 for somepkg in otherpkgnames:
 420                     curarchs.extend(Pkg(somepkg, dsttree).files.keys())
 421                 for arch in curarchs:
 422                     if arch not in pkg.files.keys():
 423                         missingarchs.append(arch)
 424                 if missingarchs:
 425                     pkg.error('moving would remove archs: %s' % self.__arch_stringify(missingarchs))
 426             else:
 427                 # warn if a package isn't built for all archs
 428                 if (config.separate_noarch and 'noarch' in pkg.files.keys() and len(pkg.files.keys()) == 2):
 429                     continue
 430                 elif len(pkg.files.keys()) != len(config.ftp_archs) + 1:
 431                     missingarchs = []
 432                     for arch in config.ftp_archs:
 433                         if arch not in pkg.files.keys():
 434                             missingarchs.append(arch)
 435                     pkg.warning('not built for archs: %s' % self.__arch_stringify(missingarchs))
 436
 437     def __rmolderfromsrc(self, test = False):
 438         for pkg in self.marked4moving:
 439             olderpkgnames = self.__find_older_pkgs(pkg)
 440             for i in olderpkgnames:
 441                 Pkg(i, self).remove(test)
 442
 443     def __rmotherfromdst(self, dsttree, test = False, archivetree = None):
 444         for pkg in self.marked4moving:
 445             pkgnames = self.__find_other_pkgs(pkg, dsttree)
 446             for i in pkgnames:
 447                 if archivetree == None:
 448                     Pkg(i, dsttree).remove(test)
 449                 else:
 450                     Pkg(i, dsttree).move(archivetree, test = test)
 451
 452     # Used more than once filter functions
 453     def __find_other_pkgs(self, pkg, tree):
 454         escapedpkgname = pkg.name.replace('.', '\.').replace('+', '\+')
 455         ziewre = re.compile(escapedpkgname + '-[^-]*-[^-]*$')
 456         def filter_other_pkgs(x):
 457             if ziewre.match(x) and not x == pkg.nvr:
 458                 return True
 459             else:
 460                 return False
 461         return filter(filter_other_pkgs, tree.pkgnames)
 462
 463     def __find_older_pkgs(self, pkg):
 464         def filter_older_pkgs(x):
 465             c = x.split('-')
 466             rc = rpm.labelCompare(('0', pkg.version, pkg.release),
 467                                                         ('0', c[-2], c[-1]))
 468             if rc == 1: # pkg > x
 469                 return True
 470             else:
 471                 return False
 472         return filter(filter_older_pkgs, self.__find_other_pkgs(pkg, self))
 473
 474     def __checksigns(self, tree, pkgs, test = False):
 475         """
 476         Checks if pkgs in tree are all signed.
 477
 478         in case of test = true, error flag is set for unsigned packages
 479         """
 480         if not tree.treename in config.signed_trees:
 481             return
 482
 483         for pkg in pkgs:
 484             unsigned = 0
 485             for file in pkg.rpmfiles():
 486                 if not is_signed(file):
 487                     unsigned += 1
 488
 489             if unsigned != 0:
 490                 if test == True:
 491                     if not quietmode:
 492                         pkg.warning('%d files not signed' % unsigned)
 493                 else:
 494                     pkg.error('%d files not signed' % unsigned)
 495
 496     def __checkforobsoletes(self, tree, pkgs, test = False):
 497         """
 498         Checks queue file if package obsoletes something in destination tree and suggest for removal.
 499
 500         Only NAME tag is compared, i.e virtual packages do not get reported.
 501
 502         """
 503         if test != True:
 504             return
 505
 506         def findbyname(name):
 507             def x(nvr):
 508                 return '-'.join(nvr.split('-')[:-2]) == name
 509             return filter(x, tree.pkgnames)
 510
 511         for pkg in pkgs:
 512             obsoletes = pkg.obsoletes()
 513             if not obsoletes:
 514                 continue
 515
 516             for pn, setlist in obsoletes.items():
 517                 for item in setlist:
 518                     p = findbyname(item)
 519                     if p:
 520                         pkg.warning('obsoletes %s (via %s) in dest tree, perhaps you want rmpkg' % (p,pn))
 521
 522     def __checkforrelease(self, tree, pkgs, test = False):
 523         """
 524         Checks queue file if package release is non integer.
 525
 526         """
 527         if test != True:
 528             return
 529
 530         for pkg in pkgs:
 531             if not pkg.is_release():
 532                 pkg.warning('non-integer release: %s' % pkg.release)