modules/ftptree.py

   1 # vi: encoding=utf-8 ts=8 sts=4 sw=4 et
   2
   3 import os, config, string, urllib, re, rpm
   4 from common import fileexists, noarchcachedir
   5 from baseftptree import BasePkg, BaseFtpTree
   6 from sign import is_signed
   7
   8 errnum = 0
   9 quietmode = False
  10
  11 class SomeError(Exception):
  12     def __init__(self):
  13         return
  14
  15     def __str__(self):
  16         print "","An Error occured!"
  17
  18 def bailoutonerror():
  19     if not errnum == 0:
  20         print "%d error(s) encountered... aborting" % errnum
  21         raise SomeError
  22
  23 def pinfo(msg):
  24     print 'INFO: ' + msg
  25
  26 def perror(msg):
  27     global errnum
  28     errnum = errnum + 1
  29     print 'ERR: ' + msg
  30
  31 def pwarning(msg):
  32     print 'WARN: ' + msg
  33
  34 def rm(file, test = False):
  35     if test:
  36         if not os.path.exists(file):
  37             pinfo("TEST os.remove(%s): file doesn't exists" % file)
  38     else:
  39         try:
  40             os.remove(file)
  41         except OSError, e:
  42             pinfo("os.remove(%s): %s" % (file, e))
  43             #raise
  44
  45 def mv(src, dst, test = False):
  46     fsrc = src
  47     fdst = dst + '/' + src.split('/')[-1]
  48     if test:
  49         if not os.path.exists(src):
  50             pinfo("TEST os.rename(%s, %s): source doesn't exists" % (fsrc, fdst))
  51         if not os.path.exists(dst):
  52             pinfo("TEST destination doesn't exist: %s" % dst)
  53     else:
  54         try:
  55             os.rename(fsrc, fdst)
  56         except OSError, e:
  57             pinfo("os.rename(%s, %s): %s" % (fsrc, fdst, e))
  58             raise
  59
  60 class Pkg(BasePkg):
  61     def __init__(self, nvr, tree):
  62         BasePkg.__init__(self, nvr, tree)
  63         self.name = string.join(nvr.split('-')[:-2], '-')
  64         self.version = nvr.split('-')[-2]
  65         self.release = nvr.split('-')[-1]
  66         self.marked4removal = False
  67         self.marked4moving = False
  68         self.marked4movingpool = []
  69         self.errors = []
  70         self.warnings = []
  71
  72     def __cmp__(self, pkg):
  73         if self.name > pkg.name:
  74             return 1
  75         elif self.name < pkg.name:
  76             return -1
  77         else:
  78             return rpm.labelCompare(('0', self.version, self.release),
  79                                     ('0', pkg.version, pkg.release))
  80
  81
  82     # unfortunately can't do new Pkg(NVR), and have no "tree" in this pkg context
  83     # so this static function
  84     def is_debuginfo(self, nvr):
  85         """
  86         returns true if NVR is debuginfo package and separate debuginfo is enabled
  87         """
  88         if not config.separate_debuginfo:
  89             return False
  90         pkg = nvr.split('-')[:-2]
  91         return pkg[-1] == 'debuginfo'
  92
  93     def is_sourcefile(self, file):
  94         """
  95         returns true if file is source package
  96         """
  97         return file[-8:] == '.src.rpm'
  98
  99     # returns true if package build is integer
 100     def is_release(self):
 101         """
 102         To account Release tags with subver macros, we consider integer release
 103         if it contains odd number of dots:
 104
 105         1 -> True
 106         0.1 -> False
 107         0.%{subver}.%{rel}, %{rel} = 1 -> 0.20010.1 -> True
 108         0.%{subver}.%{rel}, %{rel} = 0.1 -> 0.20010.0.1 -> False
 109         """
 110         return self.release.count('.') % 2 == 0
 111
 112     def mark4moving(self):
 113         if not self.marked4moving:
 114             # Only one pkg in this pool can be marked for moving
 115             for pkg in self.marked4movingpool:
 116                 pkg.unmark4moving()
 117             self.tree.marked4moving.append(self)
 118             self.marked4moving=True
 119
 120     def unmark4moving(self):
 121         if self.marked4moving:
 122             self.tree.marked4moving.remove(self)
 123             self.marked4moving=False
 124
 125     def mark4removal(self):
 126         if not self.marked4removal:
 127             self.tree.marked4removal.append(self)
 128             self.marked4removal=True
 129
 130     def error(self, msg):
 131         self.errors.append(msg)
 132         if not quietmode:
 133             perror('%s %s' % (self.nvr, msg))
 134
 135     def warning(self, msg):
 136         self.warnings.append(msg)
 137         if not quietmode:
 138             pwarning('%s %s' % (self.nvr, msg))
 139
 140     def load(self, content=None):
 141         BasePkg.load(self, content)
 142         if self.info.has_key('move'):
 143             self.mark4moving()
 144
 145     def writeinfo(self):
 146         f = open(self.tree.basedir+'/SRPMS/.metadata/'+self.nvr+'.src.rpm.info', 'w')
 147         for bid in self.build.keys():
 148             f.write("info:build:%s:requester:%s\ninfo:build:%s:requester_email:%s\n" % (bid, self.build[bid].requester, bid, self.build[bid].requester_email))
 149         for key in self.info.keys():
 150             f.write("info:%s:%s\n" % (key, string.join(self.info[key], ':')))
 151         for arch in self.files.keys():
 152             for rpm in self.files[arch]:
 153                 f.write("file:%s:%s\n" % (arch, rpm))
 154
 155     def remove(self, test = False):
 156         """
 157         Remove package from ftp
 158         """
 159         for arch in self.files.keys():
 160             for rpm in self.files[arch]:
 161                 if self.is_debuginfo(rpm):
 162                     rm(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, test)
 163                 else:
 164                     rm(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, test)
 165                 if arch == 'noarch':
 166                     if fileexists(noarchcachedir + rpm + '.filelist'):
 167                         rm(noarchcachedir + rpm + '.filelist', test)
 168                     if fileexists(noarchcachedir + rpm + '.reqlist'):
 169                         rm(noarchcachedir + rpm + '.reqlist', test)
 170         rm(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', test)
 171
 172     def rpmfiles(self, debugfiles = True, sourcefiles  = True):
 173         """
 174         Return rpm files related to this package
 175         """
 176         files = []
 177         for arch, rpms in self.files.items():
 178             for nvr in rpms:
 179                 if self.is_debuginfo(nvr):
 180                     if debugfiles:
 181                         files.append(self.tree.basedir + '/' + arch + '/debuginfo/' + nvr)
 182                 else:
 183                     if self.is_sourcefile(nvr):
 184                         if sourcefiles:
 185                             files.append(self.tree.basedir + '/' + arch + '/RPMS/' + nvr)
 186                     else:
 187                         files.append(self.tree.basedir + '/' + arch + '/RPMS/' + nvr)
 188         return files
 189
 190     def obsoletes(self):
 191         """
 192         Return obsoletes for all packages in Pkg:
 193
 194         {'php-geshi': set(['geshi'])}
 195
 196         """
 197         def rpmhdr(pkg):
 198             ts = rpm.ts()
 199             fdno = os.open(pkg, os.O_RDONLY)
 200             hdr = ts.hdrFromFdno(fdno)
 201             os.close(fdno)
 202             return hdr
 203
 204         obsoletes = {}
 205         for rpmfile in self.rpmfiles():
 206             if not os.path.exists(rpmfile):
 207                 continue
 208             hdr = rpmhdr(rpmfile)
 209             if not hdr[rpm.RPMTAG_OBSOLETES]:
 210                 continue
 211
 212             name = hdr[rpm.RPMTAG_NAME]
 213             if not name in obsoletes:
 214                 obsoletes[name] = set()
 215
 216             for tag in hdr[rpm.RPMTAG_OBSOLETES]:
 217                 obsoletes[name].add(tag)
 218
 219         return obsoletes
 220
 221     def move(self, dsttree, test = False):
 222         if dsttree.has_key(self.nvr):
 223             movedany = False
 224             for arch in self.files.keys():
 225                 if arch in dsttree[self.nvr].files.keys():
 226                     msg = ""
 227                     if test:
 228                         msg = "TEST "
 229                     pinfo("%sArch %s for %s is already present in dest tree; removing from srctree" % (msg, arch, self.nvr))
 230                     for rpm in self.files[arch]:
 231                         if self.is_debuginfo(rpm):
 232                             rm(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, test)
 233                         else:
 234                             rm(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, test)
 235                 else:
 236                     movedany = True
 237                     dsttree[self.nvr].files[arch] = self.files[arch]
 238                     for rpm in self.files[arch]:
 239                         if self.is_debuginfo(rpm):
 240                             mv(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, dsttree.basedir + '/' + arch + '/debuginfo/', test)
 241                         else:
 242                             mv(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, dsttree.basedir + '/' + arch + '/RPMS/', test)
 243             if not test and movedany:
 244                 for bid in self.build.keys():
 245                     dsttree[self.nvr].build[bid] = self.build[bid]
 246                 dsttree[self.nvr].writeinfo()
 247             rm(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', test)
 248         else:
 249             # move files
 250             for arch in self.files.keys():
 251                 for rpm in self.files[arch]:
 252                     if self.is_debuginfo(rpm):
 253                         mv(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, dsttree.basedir + '/' + arch + '/debuginfo/', test)
 254                     else:
 255                         mv(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, dsttree.basedir + '/' + arch + '/RPMS/', test)
 256
 257             # move metadata
 258             mv(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', dsttree.basedir + '/SRPMS/.metadata/', test)
 259
 260 class FtpTree(BaseFtpTree):
 261     def __init__(self, tree, loadall=False):
 262         BaseFtpTree.__init__(self, tree)
 263         self.loadedpkgs = {}
 264         self.marked4removal = []
 265         self.marked4moving = []
 266         self.pkgnames = []
 267         self.__loadpkgnames()
 268         if loadall:
 269             for pkgname in self.pkgnames:
 270                 self.loadedpkgs[pkgname] = Pkg(pkgname, self)
 271         # Tests:
 272         self.do_checkbuild = True
 273
 274     def __getitem__(self, key):
 275         if self.loadedpkgs.has_key(key):
 276             return self.loadedpkgs[key]
 277         elif key in self.pkgnames:
 278             pkg=Pkg(key, self)
 279             self.loadedpkgs[key]=pkg
 280             return pkg
 281         else:
 282             raise KeyError, key
 283
 284     def has_key(self, key):
 285         if key in self.pkgnames:
 286             return True
 287         else:
 288             return False
 289
 290     def keys(self):
 291         return self.pkgnames
 292
 293     def values(self):
 294         return self.loadedpkgs.values()
 295
 296     def checktree(self, dsttree):
 297         self.__checkbuild(self.loadedpkgs.values())
 298         self.__checkarchs(dsttree, self.loadedpkgs.values())
 299
 300     def testmove(self, dsttree, archivetree = None):
 301         self.__checkbuild(self.marked4moving)
 302         self.__checkarchs(dsttree, self.marked4moving)
 303         self.__checkduplicates(self.marked4moving)
 304
 305         self.__checksigns(dsttree, self.marked4moving, test = True)
 306         self.__checkforobsoletes(dsttree, self.marked4moving, test = True)
 307         self.__checkforrelease(dsttree, self.marked4moving, test = True)
 308
 309         if not self.treename.count("archive"):
 310             self.__rmolderfromsrc(test = True)
 311
 312         self.__rmotherfromdst(dsttree, test = True, archivetree = archivetree)
 313
 314         for pkg in self.marked4moving:
 315             pkg.move(dsttree, test = True)
 316
 317     def movepkgs(self, dsttree, archivetree = None):
 318         if self.do_checkbuild:
 319             self.__checkbuild(self.marked4moving)
 320         bailoutonerror()
 321
 322         self.__checkarchs(dsttree, self.marked4moving)
 323         bailoutonerror()
 324
 325         self.__checksigns(dsttree, self.marked4moving)
 326         bailoutonerror()
 327
 328         if not self.treename.count("archive"):
 329             self.__rmolderfromsrc()
 330
 331         self.__rmotherfromdst(dsttree, archivetree = archivetree)
 332
 333         for pkg in self.marked4moving:
 334             pkg.move(dsttree)
 335
 336     def rpmfiles(self, debugfiles = True, sourcefiles = True):
 337         if self.do_checkbuild:
 338             self.__checkbuild(self.marked4moving)
 339
 340         files = []
 341         for pkg in self.marked4moving:
 342             files += pkg.rpmfiles(debugfiles = debugfiles, sourcefiles = sourcefiles)
 343         return files
 344
 345     def removepkgs(self):
 346         if self.do_checkbuild:
 347             self.__checkbuild(self.marked4removal)
 348         bailoutonerror()
 349         for pkg in self.marked4removal:
 350             pkg.remove()
 351
 352     def mark4removal(self, wannabepkgs):
 353         self.__mark4something(wannabepkgs, Pkg.mark4removal)
 354
 355     def mark4moving(self, wannabepkgs):
 356         self.__mark4something(wannabepkgs, Pkg.mark4moving)
 357
 358
 359     # Internal functions below
 360     def __arch_stringify(self, list):
 361         ret = []
 362         dist = config.ftp_dist;
 363         for arch in list:
 364             ret.append(dist + '-' + arch)
 365         return ' '.join(ret)
 366
 367     def __loadpkgnames(self):
 368         def checkfiletype(name):
 369             if name[-13:]=='.src.rpm.info':
 370                 return True
 371             else:
 372                 return False
 373         list = filter(checkfiletype, os.listdir(self.basedir+'/SRPMS/.metadata'))
 374         self.pkgnames = map((lambda x: x[:-13]), list)
 375
 376     def __mark4something(self, wannabepkgs, markfunction):
 377         def chopoffextension(pkg):
 378             found = pkg.find('.src.rpm')
 379             if found == -1:
 380                 return pkg
 381             else:
 382                 return pkg[:found]
 383
 384         for wannabepkg in wannabepkgs:
 385             pkgname = chopoffextension(wannabepkg)
 386             if pkgname in self.pkgnames:
 387                 if not pkgname in self.loadedpkgs.keys():
 388                     self.loadedpkgs[pkgname]=Pkg(pkgname, self)
 389                 markfunction(self.loadedpkgs[pkgname])
 390             else:
 391                 perror('%s not found in source tree' % pkgname)
 392         bailoutonerror()
 393
 394     def __checkbuild(self, marked):
 395         """
 396         Checks queue file if all arches are built
 397
 398         Reads config.builderqueue to grab the info
 399         """
 400         f = urllib.urlopen(config.builderqueue)
 401         requests = {}
 402         reid = re.compile(r'^.*id=(.*) pri.*$')
 403         regb = re.compile(r'^group:.*$|builders:.*$', re.M)
 404         for i in re.findall(regb, f.read()):
 405             if i[0] == 'g':
 406                 id = reid.sub(r'\1', i)
 407                 requests[id] = ""
 408             elif i[0]=='b':
 409                 requests[id] = requests[id] + i
 410         f.close()
 411
 412         for pkg in marked:
 413             for bid in pkg.build.keys():
 414                 if requests.has_key(bid) and not requests[bid].find('?') == -1:
 415                     pkg.error("(buildid %s) building not finished" % bid)
 416
 417     def __checkarchs(self, dsttree, marked):
 418         """
 419         Checks marked pkgs it is built on all archs.
 420         """
 421         for pkg in marked:
 422             if len(pkg.files.keys()) <= 1:
 423                 pkg.error('has only src.rpm built')
 424                 continue
 425             otherpkgnames = self.__find_other_pkgs(pkg, dsttree)
 426
 427             # check if we're not removing some archs
 428             if otherpkgnames:
 429                 curarchs = []
 430                 missingarchs = []
 431                 for somepkg in otherpkgnames:
 432                     curarchs.extend(Pkg(somepkg, dsttree).files.keys())
 433                 for arch in curarchs:
 434                     if arch not in pkg.files.keys():
 435                         missingarchs.append(arch)
 436                 if missingarchs:
 437                     pkg.error('moving would remove archs: %s' % self.__arch_stringify(missingarchs))
 438             else:
 439                 # warn if a package isn't built for all archs
 440                 if (config.separate_noarch and 'noarch' in pkg.files.keys() and len(pkg.files.keys()) == 2):
 441                     continue
 442                 elif len(pkg.files.keys()) != len(config.ftp_archs) + 1:
 443                     missingarchs = []
 444                     for arch in config.ftp_archs:
 445                         if arch not in pkg.files.keys():
 446                             missingarchs.append(arch)
 447                     pkg.warning('not built for archs: %s' % self.__arch_stringify(missingarchs))
 448
 449     def __checkduplicates(self, marked):
 450         """
 451         Checks if marked packages contain duplicate packages (with different versions)
 452         """
 453         for pkg in marked:
 454             olderpkgnames = self.__find_older_pkgs(pkg)
 455             for i in olderpkgnames:
 456                 markednames = [str(x) for x in marked]
 457                 if i in markednames:
 458                     pkg.error('duplicate package: %s' % i)
 459
 460     def __rmolderfromsrc(self, test = False):
 461         for pkg in self.marked4moving:
 462             olderpkgnames = self.__find_older_pkgs(pkg)
 463             for i in olderpkgnames:
 464                 Pkg(i, self).remove(test)
 465
 466     def __rmotherfromdst(self, dsttree, test = False, archivetree = None):
 467         for pkg in self.marked4moving:
 468             pkgnames = self.__find_other_pkgs(pkg, dsttree)
 469             for i in pkgnames:
 470                 if archivetree == None:
 471                     Pkg(i, dsttree).remove(test)
 472                 else:
 473                     Pkg(i, dsttree).move(archivetree, test = test)
 474
 475     # Used more than once filter functions
 476     def __find_other_pkgs(self, pkg, tree):
 477         escapedpkgname = pkg.name.replace('.', '\.').replace('+', '\+')
 478         ziewre = re.compile(escapedpkgname + '-[^-]*-[^-]*$')
 479         def filter_other_pkgs(x):
 480             if ziewre.match(x) and not x == pkg.nvr:
 481                 return True
 482             else:
 483                 return False
 484         return filter(filter_other_pkgs, tree.pkgnames)
 485
 486     def __find_older_pkgs(self, pkg):
 487         def filter_older_pkgs(x):
 488             c = x.split('-')
 489             rc = rpm.labelCompare(('0', pkg.version, pkg.release),
 490                                                         ('0', c[-2], c[-1]))
 491             if rc == 1: # pkg > x
 492                 return True
 493             else:
 494                 return False
 495         return filter(filter_older_pkgs, self.__find_other_pkgs(pkg, self))
 496
 497     def __checksigns(self, tree, pkgs, test = False):
 498         """
 499         Checks if pkgs in tree are all signed.
 500
 501         in case of test = true, error flag is set for unsigned packages
 502         """
 503         if not tree.treename in config.signed_trees:
 504             return
 505
 506         for pkg in pkgs:
 507             unsigned = 0
 508             for file in pkg.rpmfiles():
 509                 if not is_signed(file):
 510                     unsigned += 1
 511
 512             if unsigned != 0:
 513                 if test == True:
 514                     if not quietmode:
 515                         pkg.warning('%d files not signed' % unsigned)
 516                 else:
 517                     pkg.error('%d files not signed' % unsigned)
 518
 519     def __checkforobsoletes(self, tree, pkgs, test = False):
 520         """
 521         Checks queue file if package obsoletes something in destination tree and suggest for removal.
 522
 523         Only NAME tag is compared, i.e virtual packages do not get reported.
 524
 525         """
 526         if test != True:
 527             return
 528
 529         def findbyname(name):
 530             def x(nvr):
 531                 return '-'.join(nvr.split('-')[:-2]) == name
 532             return filter(x, tree.pkgnames)
 533
 534         for pkg in pkgs:
 535             obsoletes = pkg.obsoletes()
 536             if not obsoletes:
 537                 continue
 538
 539             for pn, setlist in obsoletes.items():
 540                 for item in setlist:
 541                     p = findbyname(item)
 542                     if p:
 543                         pkg.warning('obsoletes %s (via %s) in dest tree, perhaps you want rmpkg' % (p,pn))
 544
 545     def __checkforrelease(self, tree, pkgs, test = False):
 546         """
 547         Checks queue file if package release is non integer.
 548
 549         """
 550         if test != True:
 551             return
 552
 553         for pkg in pkgs:
 554             if not pkg.is_release():
 555                 pkg.warning('non-integer release: %s' % pkg.release)