modules/ftptree.py

   1 # vi: encoding=utf-8 ts=8 sts=4 sw=4 et
   2
   3 from __future__ import print_function
   4
   5 import os, config, string, urllib, re, rpm
   6 from common import fileexists, noarchcachedir
   7 from baseftptree import BasePkg, BaseFtpTree
   8 from sign import is_signed
   9
  10 errnum = 0
  11 quietmode = False
  12
  13 class SomeError(Exception):
  14     def __init__(self):
  15         return
  16
  17     def __str__(self):
  18         return "An Error occured!"
  19
  20 def bailoutonerror():
  21     if not errnum == 0:
  22         print("%d error(s) encountered... aborting" % errnum)
  23         raise SomeError()
  24
  25 def pinfo(msg):
  26     print('INFO: ' + msg)
  27
  28 def perror(msg):
  29     global errnum
  30     errnum = errnum + 1
  31     print('ERR: ' + msg)
  32
  33 def pwarning(msg):
  34     print('WARN: ' + msg)
  35
  36 def rm(file, test = False):
  37     if test:
  38         if not os.path.exists(file):
  39             pinfo("TEST os.remove(%s): file doesn't exists" % file)
  40     else:
  41         try:
  42             os.remove(file)
  43         except OSError as e:
  44             pinfo("os.remove(%s): %s" % (file, e))
  45             #raise
  46
  47 def mv(src, dst, test = False):
  48     fsrc = src
  49     fdst = dst + '/' + src.split('/')[-1]
  50     if test:
  51         if not os.path.exists(fsrc):
  52             pinfo("TEST os.rename(%s, %s): source doesn't exists" % (fsrc, fdst))
  53         if not os.path.exists(dst):
  54             pinfo("TEST destination doesn't exist: %s" % dst)
  55     else:
  56         try:
  57             os.rename(fsrc, fdst)
  58         except OSError as e:
  59             pinfo("os.rename(%s, %s): %s" % (fsrc, fdst, e))
  60             raise
  61
  62 class Pkg(BasePkg):
  63     def __init__(self, nvr, tree):
  64         BasePkg.__init__(self, nvr, tree)
  65         self.name = string.join(nvr.split('-')[:-2], '-')
  66         self.version = nvr.split('-')[-2]
  67         self.release = nvr.split('-')[-1]
  68         self.marked4removal = False
  69         self.marked4moving = False
  70         self.marked4movingpool = []
  71         self.errors = []
  72         self.warnings = []
  73
  74     def __cmp__(self, pkg):
  75         if self.name > pkg.name:
  76             return 1
  77         elif self.name < pkg.name:
  78             return -1
  79         else:
  80             return rpm.labelCompare(('0', self.version, self.release),
  81                                     ('0', pkg.version, pkg.release))
  82
  83
  84     # unfortunately can't do new Pkg(NVR), and have no "tree" in this pkg context
  85     # so this static function
  86     def is_debuginfo(self, nvr):
  87         """
  88         returns true if NVR is debuginfo package and separate debuginfo is enabled
  89         """
  90         if not config.separate_debuginfo:
  91             return False
  92         pkg = nvr.split('-')[:-2]
  93         return pkg[-1] == 'debuginfo'
  94
  95     def is_sourcefile(self, file):
  96         """
  97         returns true if file is source package
  98         """
  99         return file[-8:] == '.src.rpm'
 100
 101     # returns true if package build is integer
 102     def is_release(self):
 103         """
 104         To account Release tags with subver macros, we consider integer release
 105         if it contains odd number of dots:
 106
 107         1 -> True
 108         0.1 -> False
 109         0.%{subver}.%{rel}, %{rel} = 1 -> 0.20010.1 -> True
 110         0.%{subver}.%{rel}, %{rel} = 0.1 -> 0.20010.0.1 -> False
 111         """
 112         return self.release.count('.') % 2 == 0
 113
 114     def mark4moving(self):
 115         if not self.marked4moving:
 116             # Only one pkg in this pool can be marked for moving
 117             for pkg in self.marked4movingpool:
 118                 pkg.unmark4moving()
 119             self.tree.marked4moving.append(self)
 120             self.marked4moving=True
 121
 122     def unmark4moving(self):
 123         if self.marked4moving:
 124             self.tree.marked4moving.remove(self)
 125             self.marked4moving=False
 126
 127     def mark4removal(self):
 128         if not self.marked4removal:
 129             self.tree.marked4removal.append(self)
 130             self.marked4removal=True
 131
 132     def error(self, msg):
 133         self.errors.append(msg)
 134         if not quietmode:
 135             perror('%s %s' % (self.nvr, msg))
 136
 137     def warning(self, msg):
 138         self.warnings.append(msg)
 139         if not quietmode:
 140             pwarning('%s %s' % (self.nvr, msg))
 141
 142     def load(self, content=None):
 143         BasePkg.load(self, content)
 144         if self.info.has_key('move'):
 145             self.mark4moving()
 146
 147     def writeinfo(self):
 148         f = open(self.tree.basedir+'/SRPMS/.metadata/'+self.nvr+'.src.rpm.info', 'w')
 149         for bid in self.build.keys():
 150             f.write("info:build:%s:requester:%s\ninfo:build:%s:requester_email:%s\n" % (bid, self.build[bid].requester, bid, self.build[bid].requester_email))
 151         for key in self.info.keys():
 152             f.write("info:%s:%s\n" % (key, string.join(self.info[key], ':')))
 153         for arch in self.files.keys():
 154             for rpm in self.files[arch]:
 155                 f.write("file:%s:%s\n" % (arch, rpm))
 156
 157     def remove(self, test = False):
 158         """
 159         Remove package from ftp
 160         """
 161         for arch in self.files.keys():
 162             for rpm in self.files[arch]:
 163                 if self.is_debuginfo(rpm):
 164                     rm(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, test)
 165                 else:
 166                     rm(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, test)
 167                 if arch == 'noarch':
 168                     if fileexists(noarchcachedir + rpm + '.filelist'):
 169                         rm(noarchcachedir + rpm + '.filelist', test)
 170                     if fileexists(noarchcachedir + rpm + '.reqlist'):
 171                         rm(noarchcachedir + rpm + '.reqlist', test)
 172         rm(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', test)
 173
 174     def rpmfiles(self, debugfiles = True, sourcefiles  = True):
 175         """
 176         Return rpm files related to this package
 177         """
 178         files = []
 179         for arch, rpms in self.files.items():
 180             for nvr in rpms:
 181                 if self.is_debuginfo(nvr):
 182                     if debugfiles:
 183                         files.append(self.tree.basedir + '/' + arch + '/debuginfo/' + nvr)
 184                 else:
 185                     if self.is_sourcefile(nvr):
 186                         if sourcefiles:
 187                             files.append(self.tree.basedir + '/' + arch + '/RPMS/' + nvr)
 188                     else:
 189                         files.append(self.tree.basedir + '/' + arch + '/RPMS/' + nvr)
 190         return files
 191
 192     def obsoletes(self):
 193         """
 194         Return obsoletes for all packages in Pkg:
 195
 196         {'php-geshi': set(['geshi'])}
 197
 198         """
 199         def rpmhdr(pkg):
 200             ts = rpm.ts()
 201             ts.setVSFlags(rpm.RPMVSF_NODSAHEADER)
 202             fdno = os.open(pkg, os.O_RDONLY)
 203             hdr = ts.hdrFromFdno(fdno)
 204             os.close(fdno)
 205             return hdr
 206
 207         obsoletes = {}
 208         for rpmfile in self.rpmfiles():
 209             if not os.path.exists(rpmfile):
 210                 continue
 211             hdr = rpmhdr(rpmfile)
 212             if not hdr[rpm.RPMTAG_OBSOLETES]:
 213                 continue
 214
 215             name = hdr[rpm.RPMTAG_NAME]
 216             if not name in obsoletes:
 217                 obsoletes[name] = set()
 218
 219             for tag in hdr[rpm.RPMTAG_OBSOLETES]:
 220                 obsoletes[name].add(tag)
 221
 222         return obsoletes
 223
 224     def move(self, dsttree, test = False):
 225         if dsttree.has_key(self.nvr):
 226             movedany = False
 227             for arch in self.files.keys():
 228                 if arch in dsttree[self.nvr].files.keys():
 229                     msg = ""
 230                     if test:
 231                         msg = "TEST "
 232                     pinfo("%sArch %s for %s is already present in dest tree; removing from srctree" % (msg, arch, self.nvr))
 233                     for rpm in self.files[arch]:
 234                         if self.is_debuginfo(rpm):
 235                             rm(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, test)
 236                         else:
 237                             rm(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, test)
 238                 else:
 239                     movedany = True
 240                     dsttree[self.nvr].files[arch] = self.files[arch]
 241                     for rpm in self.files[arch]:
 242                         if self.is_debuginfo(rpm):
 243                             mv(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, dsttree.basedir + '/' + arch + '/debuginfo/', test)
 244                         else:
 245                             mv(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, dsttree.basedir + '/' + arch + '/RPMS/', test)
 246             if not test and movedany:
 247                 for bid in self.build.keys():
 248                     dsttree[self.nvr].build[bid] = self.build[bid]
 249                 dsttree[self.nvr].writeinfo()
 250             rm(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', test)
 251         else:
 252             # move files
 253             for arch in self.files.keys():
 254                 for rpm in self.files[arch]:
 255                     if self.is_debuginfo(rpm):
 256                         mv(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, dsttree.basedir + '/' + arch + '/debuginfo/', test)
 257                     else:
 258                         mv(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, dsttree.basedir + '/' + arch + '/RPMS/', test)
 259
 260             # move metadata
 261             mv(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', dsttree.basedir + '/SRPMS/.metadata/', test)
 262
 263 class FtpTree(BaseFtpTree):
 264     def __init__(self, tree, loadall=False):
 265         BaseFtpTree.__init__(self, tree)
 266         self.loadedpkgs = {}
 267         self.marked4removal = []
 268         self.marked4moving = []
 269         self.pkgnames = []
 270         self.__loadpkgnames()
 271         if loadall:
 272             for pkgname in self.pkgnames:
 273                 self.loadedpkgs[pkgname] = Pkg(pkgname, self)
 274         # Tests:
 275         self.do_checkbuild = True
 276
 277     def __getitem__(self, key):
 278         if self.loadedpkgs.has_key(key):
 279             return self.loadedpkgs[key]
 280         elif key in self.pkgnames:
 281             pkg=Pkg(key, self)
 282             self.loadedpkgs[key]=pkg
 283             return pkg
 284         else:
 285             raise KeyError(key)
 286
 287     def has_key(self, key):
 288         if key in self.pkgnames:
 289             return True
 290         else:
 291             return False
 292
 293     def keys(self):
 294         return self.pkgnames
 295
 296     def values(self):
 297         return self.loadedpkgs.values()
 298
 299     def checktree(self, dsttree):
 300         self.__checkbuild(self.loadedpkgs.values())
 301         self.__checkarchs(dsttree, self.loadedpkgs.values())
 302
 303     def testmove(self, dsttree, archivetree = None):
 304         self.__checkbuild(self.marked4moving)
 305         self.__checkarchs(dsttree, self.marked4moving)
 306         if not dsttree.treename.count("archive"):
 307             self.__checkduplicates(self.marked4moving)
 308
 309         self.__checksigns(dsttree, self.marked4moving, test = True)
 310         self.__checkforobsoletes(dsttree, self.marked4moving, test = True)
 311         self.__checkforrelease(dsttree, self.marked4moving, test = True)
 312
 313         if not self.treename.count("archive"):
 314             self.__rmolderfromsrc(test = True)
 315         if not dsttree.treename.count("archive"):
 316             self.__rmotherfromdst(dsttree, test = True, archivetree = archivetree)
 317
 318         for pkg in self.marked4moving:
 319             pkg.move(dsttree, test = True)
 320
 321     def movepkgs(self, dsttree, archivetree = None):
 322         if self.do_checkbuild:
 323             self.__checkbuild(self.marked4moving)
 324         bailoutonerror()
 325
 326         self.__checkarchs(dsttree, self.marked4moving)
 327         bailoutonerror()
 328
 329         self.__checksigns(dsttree, self.marked4moving)
 330         bailoutonerror()
 331
 332         if not self.treename.count("archive"):
 333             self.__rmolderfromsrc()
 334         if not dsttree.treename.count("archive"):
 335             self.__rmotherfromdst(dsttree, archivetree = archivetree)
 336
 337         for pkg in self.marked4moving:
 338             pkg.move(dsttree)
 339
 340     def rpmfiles(self, debugfiles = True, sourcefiles = True):
 341         if self.do_checkbuild:
 342             self.__checkbuild(self.marked4moving)
 343
 344         files = []
 345         for pkg in self.marked4moving:
 346             files += pkg.rpmfiles(debugfiles = debugfiles, sourcefiles = sourcefiles)
 347         return files
 348
 349     def removepkgs(self):
 350         if self.do_checkbuild:
 351             self.__checkbuild(self.marked4removal)
 352         bailoutonerror()
 353         for pkg in self.marked4removal:
 354             pkg.remove()
 355
 356     def mark4removal(self, wannabepkgs):
 357         self.__mark4something(wannabepkgs, Pkg.mark4removal)
 358
 359     def mark4moving(self, wannabepkgs):
 360         self.__mark4something(wannabepkgs, Pkg.mark4moving)
 361
 362     # Internal functions below
 363     def __arch_stringify(self, list):
 364         ret = []
 365         dist = config.ftp_dist;
 366         for arch in list:
 367             ret.append(dist + '-' + arch)
 368         return ' '.join(ret)
 369
 370     def __loadpkgnames(self):
 371         def checkfiletype(name):
 372             if name[-13:]=='.src.rpm.info':
 373                 return True
 374             else:
 375                 return False
 376         list = filter(checkfiletype, os.listdir(self.basedir+'/SRPMS/.metadata'))
 377         self.pkgnames = map((lambda x: x[:-13]), list)
 378
 379     def __mark4something(self, wannabepkgs, markfunction):
 380         def chopoffextension(pkg):
 381             found = pkg.find('.src.rpm')
 382             if found == -1:
 383                 return pkg
 384             else:
 385                 return pkg[:found]
 386
 387         for wannabepkg in wannabepkgs:
 388             pkgname = chopoffextension(wannabepkg)
 389             if pkgname in self.pkgnames:
 390                 if not pkgname in self.loadedpkgs.keys():
 391                     self.loadedpkgs[pkgname]=Pkg(pkgname, self)
 392                 markfunction(self.loadedpkgs[pkgname])
 393             else:
 394                 perror('%s not found in source tree' % pkgname)
 395         bailoutonerror()
 396
 397     def __checkbuild(self, marked):
 398         """
 399         Checks queue file if all arches are built
 400
 401         Reads config.builderqueue to grab the info
 402         """
 403         f = urllib.urlopen(config.builderqueue)
 404         requests = {}
 405         reid = re.compile(r'^.*id=(.*) pri.*$')
 406         regb = re.compile(r'^group:.*$|builders:.*$', re.M)
 407         for i in re.findall(regb, f.read()):
 408             if i[0] == 'g':
 409                 id = reid.sub(r'\1', i)
 410                 requests[id] = ""
 411             elif i[0]=='b':
 412                 requests[id] = requests[id] + i
 413         f.close()
 414
 415         for pkg in marked:
 416             for bid in pkg.build.keys():
 417                 if requests.has_key(bid) and not requests[bid].find('?') == -1:
 418                     pkg.error("(buildid %s) building not finished" % bid)
 419
 420     def __checkarchs(self, dsttree, marked):
 421         """
 422         Checks marked pkgs it is built on all archs.
 423         """
 424         for pkg in marked:
 425             if len(pkg.files.keys()) <= 1:
 426                 pkg.error('has only src.rpm built')
 427                 continue
 428             otherpkgnames = self.__find_other_pkgs(pkg, dsttree)
 429
 430             # check if we're not removing some archs
 431             if otherpkgnames:
 432                 curarchs = []
 433                 missingarchs = []
 434                 for somepkg in otherpkgnames:
 435                     curarchs.extend(Pkg(somepkg, dsttree).files.keys())
 436                 for arch in curarchs:
 437                     if arch not in pkg.files.keys():
 438                         missingarchs.append(arch)
 439                 if missingarchs:
 440                     pkg.error('moving would remove archs: %s' % self.__arch_stringify(missingarchs))
 441             else:
 442                 # warn if a package isn't built for all archs
 443                 # ftp_archs + SRPMS
 444                 ftp_archs_num = len(config.ftp_archs) + 1
 445                 if (config.separate_noarch and 'noarch' in pkg.files.keys()):
 446                     # ftp_archs + SRPMS + noarch subpackages
 447                     ftp_archs_num += 1
 448                     # plain simple noarch package
 449                     if (len(pkg.files.keys()) == 2):
 450                         continue
 451
 452                 if len(pkg.files.keys()) != ftp_archs_num:
 453                     missingarchs = []
 454                     for arch in config.ftp_archs:
 455                         if arch not in pkg.files.keys():
 456                             missingarchs.append(arch)
 457                     pkg.warning('not built for archs: %s' % self.__arch_stringify(missingarchs))
 458
 459     def __checkduplicates(self, marked):
 460         """
 461         Checks if marked packages contain duplicate packages (with different versions)
 462         """
 463         for pkg in marked:
 464             olderpkgnames = self.__find_older_pkgs(pkg)
 465             for i in olderpkgnames:
 466                 markednames = [str(x) for x in marked]
 467                 if i in markednames:
 468                     pkg.error('duplicate package: %s' % i)
 469
 470     def __rmolderfromsrc(self, test = False):
 471         for pkg in self.marked4moving:
 472             olderpkgnames = self.__find_older_pkgs(pkg)
 473             for i in olderpkgnames:
 474                 Pkg(i, self).remove(test)
 475
 476     def __rmotherfromdst(self, dsttree, test = False, archivetree = None):
 477         for pkg in self.marked4moving:
 478             pkgnames = self.__find_other_pkgs(pkg, dsttree)
 479             for i in pkgnames:
 480                 if archivetree == None:
 481                     Pkg(i, dsttree).remove(test)
 482                 else:
 483                     Pkg(i, dsttree).move(archivetree, test = test)
 484
 485     # Used more than once filter functions
 486     def __find_other_pkgs(self, pkg, tree):
 487         escapedpkgname = pkg.name.replace('.', '\.').replace('+', '\+')
 488         ziewre = re.compile(escapedpkgname + '-[^-]*-[^-]*$')
 489         def filter_other_pkgs(x):
 490             if ziewre.match(x) and not x == pkg.nvr:
 491                 return True
 492             else:
 493                 return False
 494         return filter(filter_other_pkgs, tree.pkgnames)
 495
 496     def __find_older_pkgs(self, pkg):
 497         def filter_older_pkgs(x):
 498             c = x.split('-')
 499             rc = rpm.labelCompare(('0', pkg.version, pkg.release),
 500                                                         ('0', c[-2], c[-1]))
 501             if rc == 1: # pkg > x
 502                 return True
 503             else:
 504                 return False
 505         return filter(filter_older_pkgs, self.__find_other_pkgs(pkg, self))
 506
 507     def __checksigns(self, tree, pkgs, test = False):
 508         """
 509         Checks if pkgs in tree are all signed.
 510
 511         in case of test = true, error flag is set for unsigned packages
 512         """
 513         if not tree.treename in config.signed_trees:
 514             return
 515
 516         for pkg in pkgs:
 517             unsigned = 0
 518             for file in pkg.rpmfiles():
 519                 if not is_signed(file):
 520                     unsigned += 1
 521
 522             if unsigned != 0:
 523                 if test == True:
 524                     if not quietmode:
 525                         pkg.warning('%d files not signed' % unsigned)
 526                 else:
 527                     pkg.error('%d files not signed' % unsigned)
 528
 529     def __checkforobsoletes(self, tree, pkgs, test = False):
 530         """
 531         Checks queue file if package obsoletes something in destination tree and suggest for removal.
 532
 533         Only NAME tag is compared, i.e virtual packages do not get reported.
 534
 535         """
 536         if test != True:
 537             return
 538
 539         def findbyname(name):
 540             def x(nvr):
 541                 return '-'.join(nvr.split('-')[:-2]) == name
 542             return filter(x, tree.pkgnames)
 543
 544         for pkg in pkgs:
 545             obsoletes = pkg.obsoletes()
 546             if not obsoletes:
 547                 continue
 548
 549             for pn, setlist in obsoletes.items():
 550                 for item in setlist:
 551                     p = findbyname(item)
 552                     if p:
 553                         pkg.warning('obsoletes %s (via %s) in dest tree, perhaps you want rmpkg' % (p,pn))
 554
 555     def __checkforrelease(self, tree, pkgs, test = False):
 556         """
 557         Checks queue file if package release is non integer.
 558
 559         """
 560         if test != True:
 561             return
 562
 563         for pkg in pkgs:
 564             if not pkg.is_release():
 565                 pkg.warning('non-integer release: %s' % pkg.release)