modules/ftptree.py

   1 # vi: encoding=utf-8 ts=8 sts=4 sw=4 et
   2
   3 import os, config, string, urllib, re, rpm
   4 from common import fileexists, noarchcachedir
   5 from baseftptree import BasePkg, BaseFtpTree
   6 from sign import is_signed
   7
   8 errnum = 0
   9 quietmode = False
  10
  11 class SomeError(Exception):
  12     def __init__(self):
  13         return
  14
  15     def __str__(self):
  16         print "","An Error occured!"
  17
  18 def bailoutonerror():
  19     if not errnum == 0:
  20         print "%d error(s) encountered... aborting" % errnum
  21         raise SomeError
  22
  23 def pinfo(msg):
  24     print 'INFO: ' + msg
  25
  26 def perror(msg):
  27     global errnum
  28     errnum = errnum + 1
  29     print 'ERR: ' + msg
  30
  31 def pwarning(msg):
  32     print 'WARN: ' + msg
  33
  34 def rm(file, test = False):
  35     if test:
  36         if not os.path.exists(file):
  37             pinfo("TEST os.remove(%s): file doesn't exists" % file)
  38     else:
  39         try:
  40             os.remove(file)
  41         except OSError, e:
  42             pinfo("os.remove(%s): %s" % (file, e))
  43             #raise
  44
  45 def mv(src, dst, test = False):
  46     fsrc = src
  47     fdst = dst + '/' + src.split('/')[-1]
  48     if test:
  49         if not os.path.exists(fsrc):
  50             pinfo("TEST os.rename(%s, %s): source doesn't exists" % (fsrc, fdst))
  51         if not os.path.exists(dst):
  52             pinfo("TEST destination doesn't exist: %s" % dst)
  53     else:
  54         try:
  55             os.rename(fsrc, fdst)
  56         except OSError, e:
  57             pinfo("os.rename(%s, %s): %s" % (fsrc, fdst, e))
  58             raise
  59
  60 class Pkg(BasePkg):
  61     def __init__(self, nvr, tree):
  62         BasePkg.__init__(self, nvr, tree)
  63         self.name = string.join(nvr.split('-')[:-2], '-')
  64         self.version = nvr.split('-')[-2]
  65         self.release = nvr.split('-')[-1]
  66         self.marked4removal = False
  67         self.marked4moving = False
  68         self.marked4movingpool = []
  69         self.errors = []
  70         self.warnings = []
  71
  72     def __cmp__(self, pkg):
  73         if self.name > pkg.name:
  74             return 1
  75         elif self.name < pkg.name:
  76             return -1
  77         else:
  78             return rpm.labelCompare(('0', self.version, self.release),
  79                                     ('0', pkg.version, pkg.release))
  80
  81
  82     # unfortunately can't do new Pkg(NVR), and have no "tree" in this pkg context
  83     # so this static function
  84     def is_debuginfo(self, nvr):
  85         """
  86         returns true if NVR is debuginfo package and separate debuginfo is enabled
  87         """
  88         if not config.separate_debuginfo:
  89             return False
  90         pkg = nvr.split('-')[:-2]
  91         return pkg[-1] == 'debuginfo'
  92
  93     def is_sourcefile(self, file):
  94         """
  95         returns true if file is source package
  96         """
  97         return file[-8:] == '.src.rpm'
  98
  99     # returns true if package build is integer
 100     def is_release(self):
 101         """
 102         To account Release tags with subver macros, we consider integer release
 103         if it contains odd number of dots:
 104
 105         1 -> True
 106         0.1 -> False
 107         0.%{subver}.%{rel}, %{rel} = 1 -> 0.20010.1 -> True
 108         0.%{subver}.%{rel}, %{rel} = 0.1 -> 0.20010.0.1 -> False
 109         """
 110         return self.release.count('.') % 2 == 0
 111
 112     def mark4moving(self):
 113         if not self.marked4moving:
 114             # Only one pkg in this pool can be marked for moving
 115             for pkg in self.marked4movingpool:
 116                 pkg.unmark4moving()
 117             self.tree.marked4moving.append(self)
 118             self.marked4moving=True
 119
 120     def unmark4moving(self):
 121         if self.marked4moving:
 122             self.tree.marked4moving.remove(self)
 123             self.marked4moving=False
 124
 125     def mark4removal(self):
 126         if not self.marked4removal:
 127             self.tree.marked4removal.append(self)
 128             self.marked4removal=True
 129
 130     def error(self, msg):
 131         self.errors.append(msg)
 132         if not quietmode:
 133             perror('%s %s' % (self.nvr, msg))
 134
 135     def warning(self, msg):
 136         self.warnings.append(msg)
 137         if not quietmode:
 138             pwarning('%s %s' % (self.nvr, msg))
 139
 140     def load(self, content=None):
 141         BasePkg.load(self, content)
 142         if self.info.has_key('move'):
 143             self.mark4moving()
 144
 145     def writeinfo(self):
 146         f = open(self.tree.basedir+'/SRPMS/.metadata/'+self.nvr+'.src.rpm.info', 'w')
 147         for bid in self.build.keys():
 148             f.write("info:build:%s:requester:%s\ninfo:build:%s:requester_email:%s\n" % (bid, self.build[bid].requester, bid, self.build[bid].requester_email))
 149         for key in self.info.keys():
 150             f.write("info:%s:%s\n" % (key, string.join(self.info[key], ':')))
 151         for arch in self.files.keys():
 152             for rpm in self.files[arch]:
 153                 f.write("file:%s:%s\n" % (arch, rpm))
 154
 155     def remove(self, test = False):
 156         """
 157         Remove package from ftp
 158         """
 159         for arch in self.files.keys():
 160             for rpm in self.files[arch]:
 161                 if self.is_debuginfo(rpm):
 162                     rm(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, test)
 163                 else:
 164                     rm(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, test)
 165                 if arch == 'noarch':
 166                     if fileexists(noarchcachedir + rpm + '.filelist'):
 167                         rm(noarchcachedir + rpm + '.filelist', test)
 168                     if fileexists(noarchcachedir + rpm + '.reqlist'):
 169                         rm(noarchcachedir + rpm + '.reqlist', test)
 170         rm(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', test)
 171
 172     def rpmfiles(self, debugfiles = True, sourcefiles  = True):
 173         """
 174         Return rpm files related to this package
 175         """
 176         files = []
 177         for arch, rpms in self.files.items():
 178             for nvr in rpms:
 179                 if self.is_debuginfo(nvr):
 180                     if debugfiles:
 181                         files.append(self.tree.basedir + '/' + arch + '/debuginfo/' + nvr)
 182                 else:
 183                     if self.is_sourcefile(nvr):
 184                         if sourcefiles:
 185                             files.append(self.tree.basedir + '/' + arch + '/RPMS/' + nvr)
 186                     else:
 187                         files.append(self.tree.basedir + '/' + arch + '/RPMS/' + nvr)
 188         return files
 189
 190     def obsoletes(self):
 191         """
 192         Return obsoletes for all packages in Pkg:
 193
 194         {'php-geshi': set(['geshi'])}
 195
 196         """
 197         def rpmhdr(pkg):
 198             ts = rpm.ts()
 199             ts.setVSFlags(rpm.RPMVSF_NODSAHEADER)
 200             fdno = os.open(pkg, os.O_RDONLY)
 201             hdr = ts.hdrFromFdno(fdno)
 202             os.close(fdno)
 203             return hdr
 204
 205         obsoletes = {}
 206         for rpmfile in self.rpmfiles():
 207             if not os.path.exists(rpmfile):
 208                 continue
 209             hdr = rpmhdr(rpmfile)
 210             if not hdr[rpm.RPMTAG_OBSOLETES]:
 211                 continue
 212
 213             name = hdr[rpm.RPMTAG_NAME]
 214             if not name in obsoletes:
 215                 obsoletes[name] = set()
 216
 217             for tag in hdr[rpm.RPMTAG_OBSOLETES]:
 218                 obsoletes[name].add(tag)
 219
 220         return obsoletes
 221
 222     def move(self, dsttree, test = False):
 223         if dsttree.has_key(self.nvr):
 224             movedany = False
 225             for arch in self.files.keys():
 226                 if arch in dsttree[self.nvr].files.keys():
 227                     msg = ""
 228                     if test:
 229                         msg = "TEST "
 230                     pinfo("%sArch %s for %s is already present in dest tree; removing from srctree" % (msg, arch, self.nvr))
 231                     for rpm in self.files[arch]:
 232                         if self.is_debuginfo(rpm):
 233                             rm(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, test)
 234                         else:
 235                             rm(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, test)
 236                 else:
 237                     movedany = True
 238                     dsttree[self.nvr].files[arch] = self.files[arch]
 239                     for rpm in self.files[arch]:
 240                         if self.is_debuginfo(rpm):
 241                             mv(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, dsttree.basedir + '/' + arch + '/debuginfo/', test)
 242                         else:
 243                             mv(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, dsttree.basedir + '/' + arch + '/RPMS/', test)
 244             if not test and movedany:
 245                 for bid in self.build.keys():
 246                     dsttree[self.nvr].build[bid] = self.build[bid]
 247                 dsttree[self.nvr].writeinfo()
 248             rm(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', test)
 249         else:
 250             # move files
 251             for arch in self.files.keys():
 252                 for rpm in self.files[arch]:
 253                     if self.is_debuginfo(rpm):
 254                         mv(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, dsttree.basedir + '/' + arch + '/debuginfo/', test)
 255                     else:
 256                         mv(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, dsttree.basedir + '/' + arch + '/RPMS/', test)
 257
 258             # move metadata
 259             mv(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', dsttree.basedir + '/SRPMS/.metadata/', test)
 260
 261 class FtpTree(BaseFtpTree):
 262     def __init__(self, tree, loadall=False):
 263         BaseFtpTree.__init__(self, tree)
 264         self.loadedpkgs = {}
 265         self.marked4removal = []
 266         self.marked4moving = []
 267         self.pkgnames = []
 268         self.__loadpkgnames()
 269         if loadall:
 270             for pkgname in self.pkgnames:
 271                 self.loadedpkgs[pkgname] = Pkg(pkgname, self)
 272         # Tests:
 273         self.do_checkbuild = True
 274
 275     def __getitem__(self, key):
 276         if self.loadedpkgs.has_key(key):
 277             return self.loadedpkgs[key]
 278         elif key in self.pkgnames:
 279             pkg=Pkg(key, self)
 280             self.loadedpkgs[key]=pkg
 281             return pkg
 282         else:
 283             raise KeyError, key
 284
 285     def has_key(self, key):
 286         if key in self.pkgnames:
 287             return True
 288         else:
 289             return False
 290
 291     def keys(self):
 292         return self.pkgnames
 293
 294     def values(self):
 295         return self.loadedpkgs.values()
 296
 297     def checktree(self, dsttree):
 298         self.__checkbuild(self.loadedpkgs.values())
 299         self.__checkarchs(dsttree, self.loadedpkgs.values())
 300
 301     def testmove(self, dsttree, archivetree = None):
 302         self.__checkbuild(self.marked4moving)
 303         self.__checkarchs(dsttree, self.marked4moving)
 304         if not dsttree.treename.count("archive"):
 305             self.__checkduplicates(self.marked4moving)
 306
 307         self.__checksigns(dsttree, self.marked4moving, test = True)
 308         self.__checkforobsoletes(dsttree, self.marked4moving, test = True)
 309         self.__checkforrelease(dsttree, self.marked4moving, test = True)
 310
 311         if not self.treename.count("archive"):
 312             self.__rmolderfromsrc(test = True)
 313         if not dsttree.treename.count("archive"):
 314             self.__rmotherfromdst(dsttree, test = True, archivetree = archivetree)
 315
 316         for pkg in self.marked4moving:
 317             pkg.move(dsttree, test = True)
 318
 319     def movepkgs(self, dsttree, archivetree = None):
 320         if self.do_checkbuild:
 321             self.__checkbuild(self.marked4moving)
 322         bailoutonerror()
 323
 324         self.__checkarchs(dsttree, self.marked4moving)
 325         bailoutonerror()
 326
 327         self.__checksigns(dsttree, self.marked4moving)
 328         bailoutonerror()
 329
 330         if not self.treename.count("archive"):
 331             self.__rmolderfromsrc()
 332         if not dsttree.treename.count("archive"):
 333             self.__rmotherfromdst(dsttree, archivetree = archivetree)
 334
 335         for pkg in self.marked4moving:
 336             pkg.move(dsttree)
 337
 338     def rpmfiles(self, debugfiles = True, sourcefiles = True):
 339         if self.do_checkbuild:
 340             self.__checkbuild(self.marked4moving)
 341
 342         files = []
 343         for pkg in self.marked4moving:
 344             files += pkg.rpmfiles(debugfiles = debugfiles, sourcefiles = sourcefiles)
 345         return files
 346
 347     def removepkgs(self):
 348         if self.do_checkbuild:
 349             self.__checkbuild(self.marked4removal)
 350         bailoutonerror()
 351         for pkg in self.marked4removal:
 352             pkg.remove()
 353
 354     def mark4removal(self, wannabepkgs):
 355         self.__mark4something(wannabepkgs, Pkg.mark4removal)
 356
 357     def mark4moving(self, wannabepkgs):
 358         self.__mark4something(wannabepkgs, Pkg.mark4moving)
 359
 360     # Internal functions below
 361     def __arch_stringify(self, list):
 362         ret = []
 363         dist = config.ftp_dist;
 364         for arch in list:
 365             ret.append(dist + '-' + arch)
 366         return ' '.join(ret)
 367
 368     def __loadpkgnames(self):
 369         def checkfiletype(name):
 370             if name[-13:]=='.src.rpm.info':
 371                 return True
 372             else:
 373                 return False
 374         list = filter(checkfiletype, os.listdir(self.basedir+'/SRPMS/.metadata'))
 375         self.pkgnames = map((lambda x: x[:-13]), list)
 376
 377     def __mark4something(self, wannabepkgs, markfunction):
 378         def chopoffextension(pkg):
 379             found = pkg.find('.src.rpm')
 380             if found == -1:
 381                 return pkg
 382             else:
 383                 return pkg[:found]
 384
 385         for wannabepkg in wannabepkgs:
 386             pkgname = chopoffextension(wannabepkg)
 387             if pkgname in self.pkgnames:
 388                 if not pkgname in self.loadedpkgs.keys():
 389                     self.loadedpkgs[pkgname]=Pkg(pkgname, self)
 390                 markfunction(self.loadedpkgs[pkgname])
 391             else:
 392                 perror('%s not found in source tree' % pkgname)
 393         bailoutonerror()
 394
 395     def __checkbuild(self, marked):
 396         """
 397         Checks queue file if all arches are built
 398
 399         Reads config.builderqueue to grab the info
 400         """
 401         f = urllib.urlopen(config.builderqueue)
 402         requests = {}
 403         reid = re.compile(r'^.*id=(.*) pri.*$')
 404         regb = re.compile(r'^group:.*$|builders:.*$', re.M)
 405         for i in re.findall(regb, f.read()):
 406             if i[0] == 'g':
 407                 id = reid.sub(r'\1', i)
 408                 requests[id] = ""
 409             elif i[0]=='b':
 410                 requests[id] = requests[id] + i
 411         f.close()
 412
 413         for pkg in marked:
 414             for bid in pkg.build.keys():
 415                 if requests.has_key(bid) and not requests[bid].find('?') == -1:
 416                     pkg.error("(buildid %s) building not finished" % bid)
 417
 418     def __checkarchs(self, dsttree, marked):
 419         """
 420         Checks marked pkgs it is built on all archs.
 421         """
 422         for pkg in marked:
 423             if len(pkg.files.keys()) <= 1:
 424                 pkg.error('has only src.rpm built')
 425                 continue
 426             otherpkgnames = self.__find_other_pkgs(pkg, dsttree)
 427
 428             # check if we're not removing some archs
 429             if otherpkgnames:
 430                 curarchs = []
 431                 missingarchs = []
 432                 for somepkg in otherpkgnames:
 433                     curarchs.extend(Pkg(somepkg, dsttree).files.keys())
 434                 for arch in curarchs:
 435                     if arch not in pkg.files.keys():
 436                         missingarchs.append(arch)
 437                 if missingarchs:
 438                     pkg.error('moving would remove archs: %s' % self.__arch_stringify(missingarchs))
 439             else:
 440                 # warn if a package isn't built for all archs
 441                 # ftp_archs + SRPMS
 442                 ftp_archs_num = len(config.ftp_archs) + 1
 443                 if (config.separate_noarch and 'noarch' in pkg.files.keys()):
 444                     # ftp_archs + SRPMS + noarch subpackages
 445                     ftp_archs_num += 1
 446                     # plain simple noarch package
 447                     if (len(pkg.files.keys()) == 2):
 448                         continue
 449
 450                 if len(pkg.files.keys()) != ftp_archs_num:
 451                     missingarchs = []
 452                     for arch in config.ftp_archs:
 453                         if arch not in pkg.files.keys():
 454                             missingarchs.append(arch)
 455                     pkg.warning('not built for archs: %s' % self.__arch_stringify(missingarchs))
 456
 457     def __checkduplicates(self, marked):
 458         """
 459         Checks if marked packages contain duplicate packages (with different versions)
 460         """
 461         for pkg in marked:
 462             olderpkgnames = self.__find_older_pkgs(pkg)
 463             for i in olderpkgnames:
 464                 markednames = [str(x) for x in marked]
 465                 if i in markednames:
 466                     pkg.error('duplicate package: %s' % i)
 467
 468     def __rmolderfromsrc(self, test = False):
 469         for pkg in self.marked4moving:
 470             olderpkgnames = self.__find_older_pkgs(pkg)
 471             for i in olderpkgnames:
 472                 Pkg(i, self).remove(test)
 473
 474     def __rmotherfromdst(self, dsttree, test = False, archivetree = None):
 475         for pkg in self.marked4moving:
 476             pkgnames = self.__find_other_pkgs(pkg, dsttree)
 477             for i in pkgnames:
 478                 if archivetree == None:
 479                     Pkg(i, dsttree).remove(test)
 480                 else:
 481                     Pkg(i, dsttree).move(archivetree, test = test)
 482
 483     # Used more than once filter functions
 484     def __find_other_pkgs(self, pkg, tree):
 485         escapedpkgname = pkg.name.replace('.', '\.').replace('+', '\+')
 486         ziewre = re.compile(escapedpkgname + '-[^-]*-[^-]*$')
 487         def filter_other_pkgs(x):
 488             if ziewre.match(x) and not x == pkg.nvr:
 489                 return True
 490             else:
 491                 return False
 492         return filter(filter_other_pkgs, tree.pkgnames)
 493
 494     def __find_older_pkgs(self, pkg):
 495         def filter_older_pkgs(x):
 496             c = x.split('-')
 497             rc = rpm.labelCompare(('0', pkg.version, pkg.release),
 498                                                         ('0', c[-2], c[-1]))
 499             if rc == 1: # pkg > x
 500                 return True
 501             else:
 502                 return False
 503         return filter(filter_older_pkgs, self.__find_other_pkgs(pkg, self))
 504
 505     def __checksigns(self, tree, pkgs, test = False):
 506         """
 507         Checks if pkgs in tree are all signed.
 508
 509         in case of test = true, error flag is set for unsigned packages
 510         """
 511         if not tree.treename in config.signed_trees:
 512             return
 513
 514         for pkg in pkgs:
 515             unsigned = 0
 516             for file in pkg.rpmfiles():
 517                 if not is_signed(file):
 518                     unsigned += 1
 519
 520             if unsigned != 0:
 521                 if test == True:
 522                     if not quietmode:
 523                         pkg.warning('%d files not signed' % unsigned)
 524                 else:
 525                     pkg.error('%d files not signed' % unsigned)
 526
 527     def __checkforobsoletes(self, tree, pkgs, test = False):
 528         """
 529         Checks queue file if package obsoletes something in destination tree and suggest for removal.
 530
 531         Only NAME tag is compared, i.e virtual packages do not get reported.
 532
 533         """
 534         if test != True:
 535             return
 536
 537         def findbyname(name):
 538             def x(nvr):
 539                 return '-'.join(nvr.split('-')[:-2]) == name
 540             return filter(x, tree.pkgnames)
 541
 542         for pkg in pkgs:
 543             obsoletes = pkg.obsoletes()
 544             if not obsoletes:
 545                 continue
 546
 547             for pn, setlist in obsoletes.items():
 548                 for item in setlist:
 549                     p = findbyname(item)
 550                     if p:
 551                         pkg.warning('obsoletes %s (via %s) in dest tree, perhaps you want rmpkg' % (p,pn))
 552
 553     def __checkforrelease(self, tree, pkgs, test = False):
 554         """
 555         Checks queue file if package release is non integer.
 556
 557         """
 558         if test != True:
 559             return
 560
 561         for pkg in pkgs:
 562             if not pkg.is_release():
 563                 pkg.warning('non-integer release: %s' % pkg.release)