modules/ftptree.py

   1 # vi: encoding=utf-8 ts=8 sts=4 sw=4 et
   2
   3 import os, config, string, urllib, re, rpm
   4 from common import fileexists, noarchcachedir
   5 from baseftptree import BasePkg, BaseFtpTree
   6 from sign import is_signed
   7
   8 errnum = 0
   9 quietmode = False
  10
  11 class SomeError(Exception):
  12     def __init__(self):
  13         return
  14
  15     def __str__(self):
  16         print "","An Error occured!"
  17
  18 def bailoutonerror():
  19     if not errnum == 0:
  20         print "%d error(s) encountered... aborting" % errnum
  21         raise SomeError
  22
  23 def pinfo(msg):
  24     print 'INFO: ' + msg
  25
  26 def perror(msg):
  27     global errnum
  28     errnum = errnum + 1
  29     print 'ERR: ' + msg
  30
  31 def pwarning(msg):
  32     print 'WARN: ' + msg
  33
  34 def rm(file, test = False):
  35     if test:
  36         if not os.path.exists(file):
  37             pinfo("TEST os.remove(%s): file doesn't exists" % file)
  38     else:
  39         try:
  40             os.remove(file)
  41         except OSError, e:
  42             pinfo("os.remove(%s): %s" % (file, e))
  43             #raise
  44
  45 def mv(src, dst, test = False):
  46     fsrc = src
  47     fdst = dst + '/' + src.split('/')[-1]
  48     if test:
  49         if not os.path.exists(src):
  50             pinfo("TEST os.rename(%s, %s): source doesn't exists" % (fsrc, fdst))
  51         if not os.path.exists(dst):
  52             pinfo("TEST destination doesn't exist: %s" % dst)
  53     else:
  54         try:
  55             os.rename(fsrc, fdst)
  56         except OSError, e:
  57             pinfo("os.rename(%s, %s): %s" % (fsrc, fdst, e))
  58             raise
  59
  60 class Pkg(BasePkg):
  61     def __init__(self, nvr, tree):
  62         BasePkg.__init__(self, nvr, tree)
  63         self.name = string.join(nvr.split('-')[:-2], '-')
  64         self.version = nvr.split('-')[-2]
  65         self.release = nvr.split('-')[-1]
  66         self.marked4removal = False
  67         self.marked4moving = False
  68         self.marked4movingpool = []
  69         self.errors = []
  70         self.warnings = []
  71
  72     def __cmp__(self, pkg):
  73         if self.name > pkg.name:
  74             return 1
  75         elif self.name < pkg.name:
  76             return -1
  77         else:
  78             return rpm.labelCompare(('0', self.version, self.release),
  79                                     ('0', pkg.version, pkg.release))
  80
  81
  82     # unfortunately can't do new Pkg(NVR), and have no "tree" in this pkg context
  83     # so this static function
  84     def is_debuginfo(self, nvr):
  85         """
  86         returns true if NVR is debuginfo package and separate debuginfo is enabled
  87         """
  88         if not config.separate_debuginfo:
  89             return False
  90         pkg = nvr.split('-')[:-2]
  91         return pkg[-1] == 'debuginfo'
  92
  93     # returns true if package build is integer
  94     def is_release(self):
  95         """
  96         To account Release tags with subver macros, we consider integer release
  97         if it contains odd number of dots:
  98
  99         1 -> True
 100         0.1 -> False
 101         0.%{subver}.%{rel}, %{rel} = 1 -> 0.20010.1 -> True
 102         0.%{subver}.%{rel}, %{rel} = 0.1 -> 0.20010.0.1 -> False
 103         """
 104         return self.release.count('.') % 2 == 0
 105
 106     def mark4moving(self):
 107         if not self.marked4moving:
 108             # Only one pkg in this pool can be marked for moving
 109             for pkg in self.marked4movingpool:
 110                 pkg.unmark4moving()
 111             self.tree.marked4moving.append(self)
 112             self.marked4moving=True
 113
 114     def unmark4moving(self):
 115         if self.marked4moving:
 116             self.tree.marked4moving.remove(self)
 117             self.marked4moving=False
 118
 119     def mark4removal(self):
 120         if not self.marked4removal:
 121             self.tree.marked4removal.append(self)
 122             self.marked4removal=True
 123
 124     def error(self, msg):
 125         self.errors.append(msg)
 126         if not quietmode:
 127             perror('%s %s' % (self.nvr, msg))
 128
 129     def warning(self, msg):
 130         self.warnings.append(msg)
 131         if not quietmode:
 132             pwarning('%s %s' % (self.nvr, msg))
 133
 134     def load(self, content=None):
 135         BasePkg.load(self, content)
 136         if self.info.has_key('move'):
 137             self.mark4moving()
 138
 139     def writeinfo(self):
 140         f = open(self.tree.basedir+'/SRPMS/.metadata/'+self.nvr+'.src.rpm.info', 'w')
 141         for bid in self.build.keys():
 142             f.write("info:build:%s:requester:%s\ninfo:build:%s:requester_email:%s\n" % (bid, self.build[bid].requester, bid, self.build[bid].requester_email))
 143         for key in self.info.keys():
 144             f.write("info:%s:%s\n" % (key, string.join(self.info[key], ':')))
 145         for arch in self.files.keys():
 146             for rpm in self.files[arch]:
 147                 f.write("file:%s:%s\n" % (arch, rpm))
 148
 149     def remove(self, test = False):
 150         """
 151         Remove package from ftp
 152         """
 153         for arch in self.files.keys():
 154             for rpm in self.files[arch]:
 155                 if self.is_debuginfo(rpm):
 156                     rm(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, test)
 157                 else:
 158                     rm(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, test)
 159                 if arch == 'noarch':
 160                     if fileexists(noarchcachedir + rpm + '.filelist'):
 161                         rm(noarchcachedir + rpm + '.filelist', test)
 162                     if fileexists(noarchcachedir + rpm + '.reqlist'):
 163                         rm(noarchcachedir + rpm + '.reqlist', test)
 164         rm(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', test)
 165
 166     def rpmfiles(self, debugfiles = True):
 167         """
 168         Return rpm files related to this package
 169         """
 170         files = []
 171         for arch, rpms in self.files.items():
 172             for nvr in rpms:
 173                 if self.is_debuginfo(nvr):
 174                     if debugfiles:
 175                         files.append(self.tree.basedir + '/' + arch + '/debuginfo/' + nvr)
 176                 else:
 177                     files.append(self.tree.basedir + '/' + arch + '/RPMS/' + nvr)
 178         return files
 179
 180     def obsoletes(self):
 181         """
 182         Return obsoletes for all packages in Pkg:
 183
 184         {'php-geshi': set(['geshi'])}
 185
 186         """
 187         def rpmhdr(pkg):
 188             ts = rpm.ts()
 189             ts.setVSFlags(rpm._RPMVSF_NOSIGNATURES)
 190             fdno = os.open(pkg, os.O_RDONLY)
 191             hdr = ts.hdrFromFdno(fdno)
 192             os.close(fdno)
 193             return hdr
 194
 195         obsoletes = {}
 196         for rpmfile in self.rpmfiles():
 197             if not os.path.exists(rpmfile):
 198                 continue
 199             hdr = rpmhdr(rpmfile)
 200             if not hdr[rpm.RPMTAG_OBSOLETES]:
 201                 continue
 202
 203             name = hdr[rpm.RPMTAG_NAME]
 204             if not name in obsoletes:
 205                 obsoletes[name] = set()
 206
 207             for tag in hdr[rpm.RPMTAG_OBSOLETES]:
 208                 obsoletes[name].add(tag)
 209
 210         return obsoletes
 211
 212     def move(self, dsttree, test = False):
 213         if dsttree.has_key(self.nvr):
 214             movedany = False
 215             for arch in self.files.keys():
 216                 if arch in dsttree[self.nvr].files.keys():
 217                     msg = ""
 218                     if test:
 219                         msg = "TEST "
 220                     pinfo("%sArch %s for %s is already present in dest tree; removing from srctree" % (msg, arch, self.nvr))
 221                     for rpm in self.files[arch]:
 222                         if self.il-page-warningus_debuginfo(rpm):
 223                             rm(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, test)
 224                         else:
 225                             rm(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, test)
 226                 else:
 227                     movedany = True
 228                     dsttree[self.nvr].files[arch] = self.files[arch]
 229                     for rpm in self.files[arch]:
 230                         if self.is_debuginfo(rpm):
 231                             mv(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, dsttree.basedir + '/' + arch + '/debuginfo/', test)
 232                         else:
 233                             mv(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, dsttree.basedir + '/' + arch + '/RPMS/', test)
 234             if not test and movedany:
 235                 for bid in self.build.keys():
 236                     dsttree[self.nvr].build[bid] = self.build[bid]
 237                 dsttree[self.nvr].writeinfo()
 238             rm(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', test)
 239         else:
 240             # move files
 241             for arch in self.files.keys():
 242                 for rpm in self.files[arch]:
 243                     if self.is_debuginfo(rpm):
 244                         mv(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, dsttree.basedir + '/' + arch + '/debuginfo/', test)
 245                     else:
 246                         mv(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, dsttree.basedir + '/' + arch + '/RPMS/', test)
 247
 248             # move metadata
 249             mv(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', dsttree.basedir + '/SRPMS/.metadata/', test)
 250
 251 class FtpTree(BaseFtpTree):
 252     def __init__(self, tree, loadall=False):
 253         BaseFtpTree.__init__(self, tree)
 254         self.loadedpkgs = {}
 255         self.marked4removal = []
 256         self.marked4moving = []
 257         self.pkgnames = []
 258         self.__loadpkgnames()
 259         if loadall:
 260             for pkgname in self.pkgnames:
 261                 self.loadedpkgs[pkgname] = Pkg(pkgname, self)
 262         # Tests:
 263         self.do_checkbuild = True
 264
 265     def __getitem__(self, key):
 266         if self.loadedpkgs.has_key(key):
 267             return self.loadedpkgs[key]
 268         elif key in self.pkgnames:
 269             pkg=Pkg(key, self)
 270             self.loadedpkgs[key]=pkg
 271             return pkg
 272         else:
 273             raise KeyError, key
 274
 275     def has_key(self, key):
 276         if key in self.pkgnames:
 277             return True
 278         else:
 279             return False
 280
 281     def keys(self):
 282         return self.pkgnames
 283
 284     def values(self):
 285         return self.loadedpkgs.values()
 286
 287     def checktree(self, dsttree):
 288         self.__checkbuild(self.loadedpkgs.values())
 289         self.__checkarchs(dsttree, self.loadedpkgs.values())
 290
 291     def testmove(self, dsttree, archivetree = None):
 292         self.__checkbuild(self.marked4moving)
 293         self.__checkarchs(dsttree, self.marked4moving)
 294         self.__checkduplicates(self.marked4moving)
 295
 296         self.__checksigns(dsttree, self.marked4moving, test = True)
 297         self.__checkforobsoletes(dsttree, self.marked4moving, test = True)
 298         self.__checkforrelease(dsttree, self.marked4moving, test = True)
 299
 300         self.__rmolderfromsrc(test = True)
 301         self.__rmotherfromdst(dsttree, test = True, archivetree = archivetree)
 302
 303         for pkg in self.marked4moving:
 304             pkg.move(dsttree, test = True)
 305
 306     def movepkgs(self, dsttree, archivetree = None):
 307         if self.do_checkbuild:
 308             self.__checkbuild(self.marked4moving)
 309         bailoutonerror()
 310
 311         self.__checkarchs(dsttree, self.marked4moving)
 312         bailoutonerror()
 313
 314         self.__checksigns(dsttree, self.marked4moving)
 315         bailoutonerror()
 316
 317         self.__rmolderfromsrc()
 318         self.__rmotherfromdst(dsttree, archivetree = archivetree)
 319
 320         for pkg in self.marked4moving:
 321             pkg.move(dsttree)
 322
 323     def rpmfiles(self):
 324         if self.do_checkbuild:
 325             self.__checkbuild(self.marked4moving)
 326
 327         files = []
 328         for pkg in self.marked4moving:
 329             files += pkg.rpmfiles()
 330         return files
 331
 332     def removepkgs(self):
 333         if self.do_checkbuild:
 334             self.__checkbuild(self.marked4removal)
 335         bailoutonerror()
 336         for pkg in self.marked4removal:
 337             pkg.remove()
 338
 339     def mark4removal(self, wannabepkgs):
 340         self.__mark4something(wannabepkgs, Pkg.mark4removal)
 341
 342     def mark4moving(self, wannabepkgs):
 343         self.__mark4something(wannabepkgs, Pkg.mark4moving)
 344
 345
 346     # Internal functions below
 347     def __arch_stringify(self, list):
 348         ret = []
 349         # XXX: is dist line in any config?
 350         dist = 'ac'
 351         for arch in list:
 352             ret.append(dist + '-' + arch)
 353         return ' '.join(ret)
 354
 355     def __loadpkgnames(self):
 356         def checkfiletype(name):
 357             if name[-13:]=='.src.rpm.info':
 358                 return True
 359             else:
 360                 return False
 361         list = filter(checkfiletype, os.listdir(self.basedir+'/SRPMS/.metadata'))
 362         self.pkgnames = map((lambda x: x[:-13]), list)
 363
 364     def __mark4something(self, wannabepkgs, markfunction):
 365         def chopoffextension(pkg):
 366             found = pkg.find('.src.rpm')
 367             if found == -1:
 368                 return pkg
 369             else:
 370                 return pkg[:found]
 371
 372         for wannabepkg in wannabepkgs:
 373             pkgname = chopoffextension(wannabepkg)
 374             if pkgname in self.pkgnames:
 375                 if not pkgname in self.loadedpkgs.keys():
 376                     self.loadedpkgs[pkgname]=Pkg(pkgname, self)
 377                 markfunction(self.loadedpkgs[pkgname])
 378             else:
 379                 perror('%s not found in source tree' % pkgname)
 380         bailoutonerror()
 381
 382     def __checkbuild(self, marked):
 383         """
 384         Checks queue file if all arches are built
 385
 386         Reads config.builderqueue to grab the info
 387         """
 388         f = urllib.urlopen(config.builderqueue)
 389         requests = {}
 390         reid = re.compile(r'^.*id=(.*) pri.*$')
 391         regb = re.compile(r'^group:.*$|builders:.*$', re.M)
 392         for i in re.findall(regb, f.read()):
 393             if i[0] == 'g':
 394                 id = reid.sub(r'\1', i)
 395                 requests[id] = ""
 396             elif i[0]=='b':
 397                 requests[id] = requests[id] + i
 398         f.close()
 399
 400         for pkg in marked:
 401             for bid in pkg.build.keys():
 402                 if requests.has_key(bid) and not requests[bid].find('?') == -1:
 403                     pkg.error("(buildid %s) building not finished" % bid)
 404
 405     def __checkarchs(self, dsttree, marked):
 406         """
 407         Checks marked pkgs it is built on all archs.
 408         """
 409         for pkg in marked:
 410             if len(pkg.files.keys()) <= 1:
 411                 pkg.error('has only src.rpm built')
 412                 continue
 413             otherpkgnames = self.__find_other_pkgs(pkg, dsttree)
 414
 415             # check if we're not removing some archs
 416             if otherpkgnames:
 417                 curarchs = []
 418                 missingarchs = []
 419                 for somepkg in otherpkgnames:
 420                     curarchs.extend(Pkg(somepkg, dsttree).files.keys())
 421                 for arch in curarchs:
 422                     if arch not in pkg.files.keys():
 423                         missingarchs.append(arch)
 424                 if missingarchs:
 425                     pkg.error('moving would remove archs: %s' % self.__arch_stringify(missingarchs))
 426             else:
 427                 # warn if a package isn't built for all archs
 428                 if (config.separate_noarch and 'noarch' in pkg.files.keys() and len(pkg.files.keys()) == 2):
 429                     continue
 430                 elif len(pkg.files.keys()) != len(config.ftp_archs) + 1:
 431                     missingarchs = []
 432                     for arch in config.ftp_archs:
 433                         if arch not in pkg.files.keys():
 434                             missingarchs.append(arch)
 435                     pkg.warning('not built for archs: %s' % self.__arch_stringify(missingarchs))
 436
 437     def __checkduplicates(self, marked):
 438         """
 439         Checks if marked packages contain duplicate packages (with different versions)
 440         """
 441         for pkg in marked:
 442             olderpkgnames = self.__find_older_pkgs(pkg)
 443             for i in olderpkgnames:
 444                 pkg.error('duplicate package: %s' % i)
 445
 446     def __rmolderfromsrc(self, test = False):
 447         for pkg in self.marked4moving:
 448             olderpkgnames = self.__find_older_pkgs(pkg)
 449             for i in olderpkgnames:
 450                 Pkg(i, self).remove(test)
 451
 452     def __rmotherfromdst(self, dsttree, test = False, archivetree = None):
 453         for pkg in self.marked4moving:
 454             pkgnames = self.__find_other_pkgs(pkg, dsttree)
 455             for i in pkgnames:
 456                 if archivetree == None:
 457                     Pkg(i, dsttree).remove(test)
 458                 else:
 459                     Pkg(i, dsttree).move(archivetree, test = test)
 460
 461     # Used more than once filter functions
 462     def __find_other_pkgs(self, pkg, tree):
 463         escapedpkgname = pkg.name.replace('.', '\.').replace('+', '\+')
 464         ziewre = re.compile(escapedpkgname + '-[^-]*-[^-]*$')
 465         def filter_other_pkgs(x):
 466             if ziewre.match(x) and not x == pkg.nvr:
 467                 return True
 468             else:
 469                 return False
 470         return filter(filter_other_pkgs, tree.pkgnames)
 471
 472     def __find_older_pkgs(self, pkg):
 473         def filter_older_pkgs(x):
 474             c = x.split('-')
 475             rc = rpm.labelCompare(('0', pkg.version, pkg.release),
 476                                                         ('0', c[-2], c[-1]))
 477             if rc == 1: # pkg > x
 478                 return True
 479             else:
 480                 return False
 481         return filter(filter_older_pkgs, self.__find_other_pkgs(pkg, self))
 482
 483     def __checksigns(self, tree, pkgs, test = False):
 484         """
 485         Checks if pkgs in tree are all signed.
 486
 487         in case of test = true, error flag is set for unsigned packages
 488         """
 489         if not tree.treename in config.signed_trees:
 490             return
 491
 492         for pkg in pkgs:
 493             unsigned = 0
 494             for file in pkg.rpmfiles():
 495                 if not is_signed(file):
 496                     unsigned += 1
 497
 498             if unsigned != 0:
 499                 if test == True:
 500                     if not quietmode:
 501                         pkg.warning('%d files not signed' % unsigned)
 502                 else:
 503                     pkg.error('%d files not signed' % unsigned)
 504
 505     def __checkforobsoletes(self, tree, pkgs, test = False):
 506         """
 507         Checks queue file if package obsoletes something in destination tree and suggest for removal.
 508
 509         Only NAME tag is compared, i.e virtual packages do not get reported.
 510
 511         """
 512         if test != True:
 513             return
 514
 515         def findbyname(name):
 516             def x(nvr):
 517                 return '-'.join(nvr.split('-')[:-2]) == name
 518             return filter(x, tree.pkgnames)
 519
 520         for pkg in pkgs:
 521             obsoletes = pkg.obsoletes()
 522             if not obsoletes:
 523                 continue
 524
 525             for pn, setlist in obsoletes.items():
 526                 for item in setlist:
 527                     p = findbyname(item)
 528                     if p:
 529                         pkg.warning('obsoletes %s (via %s) in dest tree, perhaps you want rmpkg' % (p,pn))
 530
 531     def __checkforrelease(self, tree, pkgs, test = False):
 532         """
 533         Checks queue file if package release is non integer.
 534
 535         """
 536         if test != True:
 537             return
 538
 539         for pkg in pkgs:
 540             if not pkg.is_release():
 541                 pkg.warning('non-integer release: %s' % pkg.release)