modules/ftptree.py

   1 # vi: encoding=utf-8 ts=8 sts=4 sw=4 et
   2
   3 import os, config, string, urllib, re, rpm
   4 from common import fileexists, noarchcachedir
   5 from baseftptree import BasePkg, BaseFtpTree
   6 from sign import is_signed
   7
   8 errnum = 0
   9 quietmode = False
  10
  11 class SomeError(Exception):
  12     def __init__(self):
  13         return
  14
  15     def __str__(self):
  16         print "","An Error occured!"
  17
  18 def bailoutonerror():
  19     if not errnum == 0:
  20         print "%d error(s) encountered... aborting" % errnum
  21         raise SomeError
  22
  23 def pinfo(msg):
  24     print 'INFO: ' + msg
  25
  26 def perror(msg):
  27     global errnum
  28     errnum = errnum + 1
  29     print 'ERR: ' + msg
  30
  31 def pwarning(msg):
  32     print 'WARN: ' + msg
  33
  34 def rm(file, test = False):
  35     if test:
  36         if not os.path.exists(file):
  37             pinfo("TEST os.remove(%s): file doesn't exists" % file)
  38     else:
  39         try:
  40             os.remove(file)
  41         except OSError, e:
  42             pinfo("os.remove(%s): %s" % (file, e))
  43             #raise
  44
  45 def mv(src, dst, test = False):
  46     fsrc = src
  47     fdst = dst + '/' + src.split('/')[-1]
  48     if test:
  49         if not os.path.exists(src):
  50             pinfo("TEST os.rename(%s, %s): source doesn't exists" % (fsrc, fdst))
  51         if not os.path.exists(dst):
  52             pinfo("TEST destination doesn't exist: %s" % dst)
  53     else:
  54         try:
  55             os.rename(fsrc, fdst)
  56         except OSError, e:
  57             pinfo("os.rename(%s, %s): %s" % (fsrc, fdst, e))
  58             raise
  59
  60 class Pkg(BasePkg):
  61     def __init__(self, nvr, tree):
  62         BasePkg.__init__(self, nvr, tree)
  63         self.name = string.join(nvr.split('-')[:-2], '-')
  64         self.version = nvr.split('-')[-2]
  65         self.release = nvr.split('-')[-1]
  66         self.marked4removal = False
  67         self.marked4moving = False
  68         self.marked4movingpool = []
  69         self.errors = []
  70         self.warnings = []
  71
  72     def __cmp__(self, pkg):
  73         if self.name > pkg.name:
  74             return 1
  75         elif self.name < pkg.name:
  76             return -1
  77         else:
  78             return rpm.labelCompare(('0', self.version, self.release),
  79                                     ('0', pkg.version, pkg.release))
  80
  81
  82     # unfortunately can't do new Pkg(NVR), and have no "tree" in this pkg context
  83     # so this static function
  84     def is_debuginfo(self, nvr):
  85         """
  86         returns true if NVR is debuginfo package and separate debuginfo is enabled
  87         """
  88         if not config.separate_debuginfo:
  89             return False
  90         pkg = nvr.split('-')[:-2]
  91         return pkg[-1] == 'debuginfo'
  92
  93     def mark4moving(self):
  94         if not self.marked4moving:
  95             # Only one pkg in this pool can be marked for moving
  96             for pkg in self.marked4movingpool:
  97                 pkg.unmark4moving()
  98             self.tree.marked4moving.append(self)
  99             self.marked4moving=True
 100
 101     def unmark4moving(self):
 102         if self.marked4moving:
 103             self.tree.marked4moving.remove(self)
 104             self.marked4moving=False
 105
 106     def mark4removal(self):
 107         if not self.marked4removal:
 108             self.tree.marked4removal.append(self)
 109             self.marked4removal=True
 110
 111     def error(self, msg):
 112         self.errors.append(msg)
 113         if not quietmode:
 114             perror('%s %s' % (self.nvr, msg))
 115
 116     def warning(self, msg):
 117         self.warnings.append(msg)
 118         if not quietmode:
 119             pwarning('%s %s' % (self.nvr, msg))
 120
 121     def load(self, content=None):
 122         BasePkg.load(self, content)
 123         if self.info.has_key('move'):
 124             self.mark4moving()
 125
 126     def writeinfo(self):
 127         f = open(self.tree.basedir+'/SRPMS/.metadata/'+self.nvr+'.src.rpm.info', 'w')
 128         for bid in self.build.keys():
 129             f.write("info:build:%s:requester:%s\ninfo:build:%s:requester_email:%s\n" % (bid, self.build[bid].requester, bid, self.build[bid].requester_email))
 130         for key in self.info.keys():
 131             f.write("info:%s:%s\n" % (key, string.join(self.info[key], ':')))
 132         for arch in self.files.keys():
 133             for rpm in self.files[arch]:
 134                 f.write("file:%s:%s\n" % (arch, rpm))
 135
 136     def remove(self, test = False):
 137         """
 138         Remove package from ftp
 139         """
 140         for arch in self.files.keys():
 141             for rpm in self.files[arch]:
 142                 if self.is_debuginfo(rpm):
 143                     rm(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, test)
 144                 else:
 145                     rm(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, test)
 146                 if arch == 'noarch':
 147                     if fileexists(noarchcachedir + rpm + '.filelist'):
 148                         rm(noarchcachedir + rpm + '.filelist', test)
 149                     if fileexists(noarchcachedir + rpm + '.reqlist'):
 150                         rm(noarchcachedir + rpm + '.reqlist', test)
 151         rm(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', test)
 152
 153     def rpmfiles(self):
 154         """
 155         Return rpm files related to this package
 156         """
 157         files = []
 158         for arch, rpms in self.files.items():
 159             for nvr in rpms:
 160                 if self.is_debuginfo(nvr):
 161                     files.append(self.tree.basedir + '/' + arch + '/debuginfo/' + nvr)
 162                 else:
 163                     files.append(self.tree.basedir + '/' + arch + '/RPMS/' + nvr)
 164         return files
 165
 166     def obsoletes(self):
 167         """
 168         Return obsoletes for all packages in Pkg:
 169
 170         {'php-geshi': set(['geshi'])}
 171
 172         """
 173         def rpmhdr(pkg):
 174             ts = rpm.ts()
 175             ts.setVSFlags(rpm._RPMVSF_NOSIGNATURES)
 176             fdno = os.open(pkg, os.O_RDONLY)
 177             hdr = ts.hdrFromFdno(fdno)
 178             os.close(fdno)
 179             return hdr
 180
 181         obsoletes = {}
 182         for rpmfile in self.rpmfiles():
 183             if not os.path.exists(rpmfile):
 184                 continue
 185             hdr = rpmhdr(rpmfile)
 186             if not hdr[rpm.RPMTAG_OBSOLETES]:
 187                 continue
 188
 189             name = hdr[rpm.RPMTAG_NAME]
 190             if not name in obsoletes:
 191                 obsoletes[name] = set()
 192
 193             for tag in hdr[rpm.RPMTAG_OBSOLETES]:
 194                 obsoletes[name].add(tag)
 195
 196         return obsoletes
 197
 198     def move(self, dsttree, test = False):
 199         if dsttree.has_key(self.nvr):
 200             movedany = False
 201             for arch in self.files.keys():
 202                 if arch in dsttree[self.nvr].files.keys():
 203                     msg = ""
 204                     if test:
 205                         msg = "TEST "
 206                     pinfo("%sArch %s for %s is already present in dest tree; removing from srctree" % (msg, arch, self.nvr))
 207                     for rpm in self.files[arch]:
 208                         if self.is_debuginfo(rpm):
 209                             rm(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, test)
 210                         else:
 211                             rm(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, test)
 212                 else:
 213                     movedany = True
 214                     dsttree[self.nvr].files[arch] = self.files[arch]
 215                     for rpm in self.files[arch]:
 216                         if self.is_debuginfo(rpm):
 217                             mv(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, dsttree.basedir + '/' + arch + '/debuginfo/', test)
 218                         else:
 219                             mv(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, dsttree.basedir + '/' + arch + '/RPMS/', test)
 220             if not test and movedany:
 221                 for bid in self.build.keys():
 222                     dsttree[self.nvr].build[bid] = self.build[bid]
 223                 dsttree[self.nvr].writeinfo()
 224             rm(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', test)
 225         else:
 226             # move files
 227             for arch in self.files.keys():
 228                 for rpm in self.files[arch]:
 229                     if self.is_debuginfo(rpm):
 230                         mv(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, dsttree.basedir + '/' + arch + '/debuginfo/', test)
 231                     else:
 232                         mv(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, dsttree.basedir + '/' + arch + '/RPMS/', test)
 233
 234             # move metadata
 235             mv(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', dsttree.basedir + '/SRPMS/.metadata/', test)
 236
 237 class FtpTree(BaseFtpTree):
 238     def __init__(self, tree, loadall=False):
 239         BaseFtpTree.__init__(self, tree)
 240         self.loadedpkgs = {}
 241         self.marked4removal = []
 242         self.marked4moving = []
 243         self.pkgnames = []
 244         self.__loadpkgnames()
 245         if loadall:
 246             for pkgname in self.pkgnames:
 247                 self.loadedpkgs[pkgname] = Pkg(pkgname, self)
 248         # Tests:
 249         self.do_checkbuild = True
 250
 251     def __getitem__(self, key):
 252         if self.loadedpkgs.has_key(key):
 253             return self.loadedpkgs[key]
 254         elif key in self.pkgnames:
 255             pkg=Pkg(key, self)
 256             self.loadedpkgs[key]=pkg
 257             return pkg
 258         else:
 259             raise KeyError, key
 260
 261     def has_key(self, key):
 262         if key in self.pkgnames:
 263             return True
 264         else:
 265             return False
 266
 267     def keys(self):
 268         return self.pkgnames
 269
 270     def values(self):
 271         return self.loadedpkgs.values()
 272
 273     def checktree(self, dsttree):
 274         self.__checkbuild(self.loadedpkgs.values())
 275         self.__checkarchs(dsttree, self.loadedpkgs.values())
 276
 277     def testmove(self, dsttree, archivetree = None):
 278         self.__checkbuild(self.marked4moving)
 279         self.__checkarchs(dsttree, self.marked4moving)
 280
 281         self.__checksigns(dsttree, self.marked4moving, test = True)
 282         self.__checkforobsoletes(dsttree, self.marked4moving, test = True)
 283
 284         self.__rmolderfromsrc(test = True)
 285         self.__rmotherfromdst(dsttree, test = True, archivetree = archivetree)
 286
 287         for pkg in self.marked4moving:
 288             pkg.move(dsttree, test = True)
 289
 290     def movepkgs(self, dsttree, archivetree = None):
 291         if self.do_checkbuild:
 292             self.__checkbuild(self.marked4moving)
 293         bailoutonerror()
 294
 295         self.__checkarchs(dsttree, self.marked4moving)
 296         bailoutonerror()
 297
 298         self.__checksigns(dsttree, self.marked4moving)
 299         bailoutonerror()
 300
 301         self.__rmolderfromsrc()
 302         self.__rmotherfromdst(dsttree, archivetree = archivetree)
 303
 304         for pkg in self.marked4moving:
 305             pkg.move(dsttree)
 306
 307     def rpmfiles(self):
 308         if self.do_checkbuild:
 309             self.__checkbuild(self.marked4moving)
 310
 311         files = []
 312         for pkg in self.marked4moving:
 313             files += pkg.rpmfiles()
 314         return files
 315
 316     def removepkgs(self):
 317         if self.do_checkbuild:
 318             self.__checkbuild(self.marked4removal)
 319         bailoutonerror()
 320         for pkg in self.marked4removal:
 321             pkg.remove()
 322
 323     def mark4removal(self, wannabepkgs):
 324         self.__mark4something(wannabepkgs, Pkg.mark4removal)
 325
 326     def mark4moving(self, wannabepkgs):
 327         self.__mark4something(wannabepkgs, Pkg.mark4moving)
 328
 329
 330     # Internal functions below
 331
 332     def __loadpkgnames(self):
 333         def checkfiletype(name):
 334             if name[-13:]=='.src.rpm.info':
 335                 return True
 336             else:
 337                 return False
 338         list = filter(checkfiletype, os.listdir(self.basedir+'/SRPMS/.metadata'))
 339         self.pkgnames = map((lambda x: x[:-13]), list)
 340
 341     def __mark4something(self, wannabepkgs, markfunction):
 342         def chopoffextension(pkg):
 343             found = pkg.find('.src.rpm')
 344             if found == -1:
 345                 return pkg
 346             else:
 347                 return pkg[:found]
 348
 349         for wannabepkg in wannabepkgs:
 350             pkgname = chopoffextension(wannabepkg)
 351             if pkgname in self.pkgnames:
 352                 if not pkgname in self.loadedpkgs.keys():
 353                     self.loadedpkgs[pkgname]=Pkg(pkgname, self)
 354                 markfunction(self.loadedpkgs[pkgname])
 355             else:
 356                 perror('%s not found in source tree' % pkgname)
 357         bailoutonerror()
 358
 359     def __checkbuild(self, marked):
 360         """
 361         Checks queue file if all arches are built
 362
 363         Reads config.builderqueue to grab the info
 364         """
 365         f = urllib.urlopen(config.builderqueue)
 366         requests = {}
 367         reid = re.compile(r'^.*id=(.*) pri.*$')
 368         regb = re.compile(r'^group:.*$|builders:.*$', re.M)
 369         for i in re.findall(regb, f.read()):
 370             if i[0] == 'g':
 371                 id = reid.sub(r'\1', i)
 372                 requests[id] = ""
 373             elif i[0]=='b':
 374                 requests[id] = requests[id] + i
 375         f.close()
 376
 377         for pkg in marked:
 378             for bid in pkg.build.keys():
 379                 if requests.has_key(bid) and not requests[bid].find('?') == -1:
 380                     pkg.error("(buildid %s) building not finished" % bid)
 381
 382     def __checkarchs(self, dsttree, marked):
 383         """
 384         Checks marked pkgs it is built on all archs.
 385         """
 386         for pkg in marked:
 387             if len(pkg.files.keys()) <= 1:
 388                 pkg.error('has only src.rpm built')
 389                 continue
 390             otherpkgnames = self.__find_other_pkgs(pkg, dsttree)
 391
 392             # check if we're not removing some archs
 393             if otherpkgnames:
 394                 curarchs = []
 395                 missingarchs = []
 396                 for somepkg in otherpkgnames:
 397                     curarchs.extend(Pkg(somepkg, dsttree).files.keys())
 398                 for arch in curarchs:
 399                     if arch not in pkg.files.keys():
 400                         missingarchs.append(arch)
 401                 if missingarchs:
 402                     pkg.error('moving would remove archs: %s' % missingarchs)
 403             else:
 404                 # warn if a package isn't built for all archs
 405                 if (config.separate_noarch and 'noarch' in pkg.files.keys() and len(pkg.files.keys()) == 2):
 406                     continue
 407                 elif len(pkg.files.keys()) != len(config.ftp_archs) + 1:
 408                     missingarchs = []
 409                     for arch in config.ftp_archs:
 410                         if arch not in pkg.files.keys():
 411                             missingarchs.append(arch)
 412                     pkg.warning('not built for archs: %s' % missingarchs)
 413
 414     def __rmolderfromsrc(self, test = False):
 415         for pkg in self.marked4moving:
 416             olderpkgnames = self.__find_older_pkgs(pkg)
 417             for i in olderpkgnames:
 418                 Pkg(i, self).remove(test)
 419
 420     def __rmotherfromdst(self, dsttree, test = False, archivetree = None):
 421         for pkg in self.marked4moving:
 422             pkgnames = self.__find_other_pkgs(pkg, dsttree)
 423             for i in pkgnames:
 424                 if archivetree == None:
 425                     Pkg(i, dsttree).remove(test)
 426                 else:
 427                     Pkg(i, dsttree).move(archivetree, test = test)
 428
 429     # Used more than once filter functions
 430     def __find_other_pkgs(self, pkg, tree):
 431         escapedpkgname = pkg.name.replace('.', '\.').replace('+', '\+')
 432         ziewre = re.compile(escapedpkgname + '-[^-]*-[^-]*$')
 433         def filter_other_pkgs(x):
 434             if ziewre.match(x) and not x == pkg.nvr:
 435                 return True
 436             else:
 437                 return False
 438         return filter(filter_other_pkgs, tree.pkgnames)
 439
 440     def __find_older_pkgs(self, pkg):
 441         def filter_older_pkgs(x):
 442             c = x.split('-')
 443             rc = rpm.labelCompare(('0', pkg.version, pkg.release),
 444                                                         ('0', c[-2], c[-1]))
 445             if rc == 1: # pkg > x
 446                 return True
 447             else:
 448                 return False
 449         return filter(filter_older_pkgs, self.__find_other_pkgs(pkg, self))
 450
 451     def __checksigns(self, tree, pkgs, test = False):
 452         """
 453         Checks if pkgs in tree are all signed.
 454
 455         in case of test = true, error flag is set for unsigned packages
 456         """
 457         if not tree.treename in config.signed_trees:
 458             return
 459
 460         for pkg in pkgs:
 461             unsigned = 0
 462             for file in pkg.rpmfiles():
 463                 if not is_signed(file):
 464                     unsigned += 1
 465
 466             if unsigned != 0:
 467                 if test == True:
 468                     if not quietmode:
 469                         pkg.warning('%d files not signed' % unsigned)
 470                 else:
 471                     pkg.error('%d files not signed' % unsigned)
 472
 473     def __checkforobsoletes(self, tree, pkgs, test = False):
 474         """
 475         Checks queue file if package obsoletes something in destination tree and suggest for removal.
 476
 477         Only NAME tag is compared, i.e virtual packages do not get reported.
 478
 479         """
 480         if test != True:
 481             return
 482
 483         def findbyname(name):
 484             def x(nvr):
 485                 return '-'.join(nvr.split('-')[:-2]) == name
 486             return filter(x, tree.pkgnames)
 487
 488         for pkg in pkgs:
 489             obsoletes = pkg.obsoletes()
 490             if not obsoletes:
 491                 continue
 492
 493             for pn, setlist in obsoletes.items():
 494                 for item in setlist:
 495                     p = findbyname(item)
 496                     if p:
 497                         pkg.warning('obsoletes %s (via %s) in dest tree, perhaps you want rmpkg' % (p,pn))