modules/ftptree.py

   1 # vi: encoding=utf-8 ts=8 sts=4 sw=4 et
   2
   3 import os, config, string, urllib, re, rpm
   4 from common import fileexists, noarchcachedir
   5 from baseftptree import BasePkg, BaseFtpTree
   6 from sign import is_signed
   7
   8 errnum = 0
   9 quietmode = False
  10
  11 class SomeError(Exception):
  12     def __init__(self):
  13         return
  14
  15     def __str__(self):
  16         print "","An Error occured!"
  17
  18 def bailoutonerror():
  19     if not errnum == 0:
  20         print "%d error(s) encountered... aborting" % errnum
  21         raise SomeError
  22
  23 def pinfo(msg):
  24     print 'INFO: ' + msg
  25
  26 def perror(msg):
  27     global errnum
  28     errnum = errnum + 1
  29     print 'ERR: ' + msg
  30
  31 def pwarning(msg):
  32     print 'WARN: ' + msg
  33
  34 def rm(file, test = False):
  35     if test:
  36         if not os.path.exists(file):
  37             pinfo("TEST os.remove(%s): file doesn't exists" % file)
  38     else:
  39         try:
  40             os.remove(file)
  41         except OSError, e:
  42             pinfo("os.remove(%s): %s" % (file, e))
  43             #raise
  44
  45 def mv(src, dst, test = False):
  46     fsrc = src
  47     fdst = dst + '/' + src.split('/')[-1]
  48     if test:
  49         if not os.path.exists(src):
  50             pinfo("TEST os.rename(%s, %s): source doesn't exists" % (fsrc, fdst))
  51         if not os.path.exists(dst):
  52             pinfo("TEST destination doesn't exist: %s" % dst)
  53     else:
  54         try:
  55             os.rename(fsrc, fdst)
  56         except OSError, e:
  57             pinfo("os.rename(%s, %s): %s" % (fsrc, fdst, e))
  58             raise
  59
  60 class Pkg(BasePkg):
  61     def __init__(self, nvr, tree):
  62         BasePkg.__init__(self, nvr, tree)
  63         self.name = string.join(nvr.split('-')[:-2], '-')
  64         self.version = nvr.split('-')[-2]
  65         self.release = nvr.split('-')[-1]
  66         self.marked4removal = False
  67         self.marked4moving = False
  68         self.marked4movingpool = []
  69         self.errors = []
  70         self.warnings = []
  71
  72     def __cmp__(self, pkg):
  73         if self.name > pkg.name:
  74             return 1
  75         elif self.name < pkg.name:
  76             return -1
  77         else:
  78             return rpm.labelCompare(('0', self.version, self.release),
  79                                     ('0', pkg.version, pkg.release))
  80
  81
  82     # unfortunately can't do new Pkg(NVR), and have no "tree" in this pkg context
  83     # so this static function
  84     def is_debuginfo(self, nvr):
  85         """
  86         returns true if NVR is debuginfo package and separate debuginfo is enabled
  87         """
  88         if not config.separate_debuginfo:
  89             return False
  90         pkg = nvr.split('-')[:-2]
  91         return pkg[-1] == 'debuginfo'
  92
  93     # returns true if package build is integer
  94     def is_release(self):
  95         """
  96         To account Release tags with subver macros, we consider integer release
  97         if it contains odd number of dots:
  98
  99         1 -> True
 100         0.1 -> False
 101         0.%{subver}.%{rel}, %{rel} = 1 -> 0.20010.1 -> True
 102         0.%{subver}.%{rel}, %{rel} = 0.1 -> 0.20010.0.1 -> False
 103         """
 104         return self.release.count('.') % 2 == 0
 105
 106     def mark4moving(self):
 107         if not self.marked4moving:
 108             # Only one pkg in this pool can be marked for moving
 109             for pkg in self.marked4movingpool:
 110                 pkg.unmark4moving()
 111             self.tree.marked4moving.append(self)
 112             self.marked4moving=True
 113
 114     def unmark4moving(self):
 115         if self.marked4moving:
 116             self.tree.marked4moving.remove(self)
 117             self.marked4moving=False
 118
 119     def mark4removal(self):
 120         if not self.marked4removal:
 121             self.tree.marked4removal.append(self)
 122             self.marked4removal=True
 123
 124     def error(self, msg):
 125         self.errors.append(msg)
 126         if not quietmode:
 127             perror('%s %s' % (self.nvr, msg))
 128
 129     def warning(self, msg):
 130         self.warnings.append(msg)
 131         if not quietmode:
 132             pwarning('%s %s' % (self.nvr, msg))
 133
 134     def load(self, content=None):
 135         BasePkg.load(self, content)
 136         if self.info.has_key('move'):
 137             self.mark4moving()
 138
 139     def writeinfo(self):
 140         f = open(self.tree.basedir+'/SRPMS/.metadata/'+self.nvr+'.src.rpm.info', 'w')
 141         for bid in self.build.keys():
 142             f.write("info:build:%s:requester:%s\ninfo:build:%s:requester_email:%s\n" % (bid, self.build[bid].requester, bid, self.build[bid].requester_email))
 143         for key in self.info.keys():
 144             f.write("info:%s:%s\n" % (key, string.join(self.info[key], ':')))
 145         for arch in self.files.keys():
 146             for rpm in self.files[arch]:
 147                 f.write("file:%s:%s\n" % (arch, rpm))
 148
 149     def remove(self, test = False):
 150         """
 151         Remove package from ftp
 152         """
 153         for arch in self.files.keys():
 154             for rpm in self.files[arch]:
 155                 if self.is_debuginfo(rpm):
 156                     rm(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, test)
 157                 else:
 158                     rm(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, test)
 159                 if arch == 'noarch':
 160                     if fileexists(noarchcachedir + rpm + '.filelist'):
 161                         rm(noarchcachedir + rpm + '.filelist', test)
 162                     if fileexists(noarchcachedir + rpm + '.reqlist'):
 163                         rm(noarchcachedir + rpm + '.reqlist', test)
 164         rm(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', test)
 165
 166     def rpmfiles(self):
 167         """
 168         Return rpm files related to this package
 169         """
 170         files = []
 171         for arch, rpms in self.files.items():
 172             for nvr in rpms:
 173                 if self.is_debuginfo(nvr):
 174                     files.append(self.tree.basedir + '/' + arch + '/debuginfo/' + nvr)
 175                 else:
 176                     files.append(self.tree.basedir + '/' + arch + '/RPMS/' + nvr)
 177         return files
 178
 179     def obsoletes(self):
 180         """
 181         Return obsoletes for all packages in Pkg:
 182
 183         {'php-geshi': set(['geshi'])}
 184
 185         """
 186         def rpmhdr(pkg):
 187             ts = rpm.ts()
 188             ts.setVSFlags(rpm._RPMVSF_NOSIGNATURES)
 189             fdno = os.open(pkg, os.O_RDONLY)
 190             hdr = ts.hdrFromFdno(fdno)
 191             os.close(fdno)
 192             return hdr
 193
 194         obsoletes = {}
 195         for rpmfile in self.rpmfiles():
 196             if not os.path.exists(rpmfile):
 197                 continue
 198             hdr = rpmhdr(rpmfile)
 199             if not hdr[rpm.RPMTAG_OBSOLETES]:
 200                 continue
 201
 202             name = hdr[rpm.RPMTAG_NAME]
 203             if not name in obsoletes:
 204                 obsoletes[name] = set()
 205
 206             for tag in hdr[rpm.RPMTAG_OBSOLETES]:
 207                 obsoletes[name].add(tag)
 208
 209         return obsoletes
 210
 211     def move(self, dsttree, test = False):
 212         if dsttree.has_key(self.nvr):
 213             movedany = False
 214             for arch in self.files.keys():
 215                 if arch in dsttree[self.nvr].files.keys():
 216                     msg = ""
 217                     if test:
 218                         msg = "TEST "
 219                     pinfo("%sArch %s for %s is already present in dest tree; removing from srctree" % (msg, arch, self.nvr))
 220                     for rpm in self.files[arch]:
 221                         if self.is_debuginfo(rpm):
 222                             rm(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, test)
 223                         else:
 224                             rm(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, test)
 225                 else:
 226                     movedany = True
 227                     dsttree[self.nvr].files[arch] = self.files[arch]
 228                     for rpm in self.files[arch]:
 229                         if self.is_debuginfo(rpm):
 230                             mv(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, dsttree.basedir + '/' + arch + '/debuginfo/', test)
 231                         else:
 232                             mv(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, dsttree.basedir + '/' + arch + '/RPMS/', test)
 233             if not test and movedany:
 234                 for bid in self.build.keys():
 235                     dsttree[self.nvr].build[bid] = self.build[bid]
 236                 dsttree[self.nvr].writeinfo()
 237             rm(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', test)
 238         else:
 239             # move files
 240             for arch in self.files.keys():
 241                 for rpm in self.files[arch]:
 242                     if self.is_debuginfo(rpm):
 243                         mv(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, dsttree.basedir + '/' + arch + '/debuginfo/', test)
 244                     else:
 245                         mv(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, dsttree.basedir + '/' + arch + '/RPMS/', test)
 246
 247             # move metadata
 248             mv(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', dsttree.basedir + '/SRPMS/.metadata/', test)
 249
 250 class FtpTree(BaseFtpTree):
 251     def __init__(self, tree, loadall=False):
 252         BaseFtpTree.__init__(self, tree)
 253         self.loadedpkgs = {}
 254         self.marked4removal = []
 255         self.marked4moving = []
 256         self.pkgnames = []
 257         self.__loadpkgnames()
 258         if loadall:
 259             for pkgname in self.pkgnames:
 260                 self.loadedpkgs[pkgname] = Pkg(pkgname, self)
 261         # Tests:
 262         self.do_checkbuild = True
 263
 264     def __getitem__(self, key):
 265         if self.loadedpkgs.has_key(key):
 266             return self.loadedpkgs[key]
 267         elif key in self.pkgnames:
 268             pkg=Pkg(key, self)
 269             self.loadedpkgs[key]=pkg
 270             return pkg
 271         else:
 272             raise KeyError, key
 273
 274     def has_key(self, key):
 275         if key in self.pkgnames:
 276             return True
 277         else:
 278             return False
 279
 280     def keys(self):
 281         return self.pkgnames
 282
 283     def values(self):
 284         return self.loadedpkgs.values()
 285
 286     def checktree(self, dsttree):
 287         self.__checkbuild(self.loadedpkgs.values())
 288         self.__checkarchs(dsttree, self.loadedpkgs.values())
 289
 290     def testmove(self, dsttree, archivetree = None):
 291         self.__checkbuild(self.marked4moving)
 292         self.__checkarchs(dsttree, self.marked4moving)
 293
 294         self.__checksigns(dsttree, self.marked4moving, test = True)
 295         self.__checkforobsoletes(dsttree, self.marked4moving, test = True)
 296         self.__checkforrelease(dsttree, self.marked4moving, test = True)
 297
 298         self.__rmolderfromsrc(test = True)
 299         self.__rmotherfromdst(dsttree, test = True, archivetree = archivetree)
 300
 301         for pkg in self.marked4moving:
 302             pkg.move(dsttree, test = True)
 303
 304     def movepkgs(self, dsttree, archivetree = None):
 305         if self.do_checkbuild:
 306             self.__checkbuild(self.marked4moving)
 307         bailoutonerror()
 308
 309         self.__checkarchs(dsttree, self.marked4moving)
 310         bailoutonerror()
 311
 312         self.__checksigns(dsttree, self.marked4moving)
 313         bailoutonerror()
 314
 315         self.__rmolderfromsrc()
 316         self.__rmotherfromdst(dsttree, archivetree = archivetree)
 317
 318         for pkg in self.marked4moving:
 319             pkg.move(dsttree)
 320
 321     def rpmfiles(self):
 322         if self.do_checkbuild:
 323             self.__checkbuild(self.marked4moving)
 324
 325         files = []
 326         for pkg in self.marked4moving:
 327             files += pkg.rpmfiles()
 328         return files
 329
 330     def removepkgs(self):
 331         if self.do_checkbuild:
 332             self.__checkbuild(self.marked4removal)
 333         bailoutonerror()
 334         for pkg in self.marked4removal:
 335             pkg.remove()
 336
 337     def mark4removal(self, wannabepkgs):
 338         self.__mark4something(wannabepkgs, Pkg.mark4removal)
 339
 340     def mark4moving(self, wannabepkgs):
 341         self.__mark4something(wannabepkgs, Pkg.mark4moving)
 342
 343
 344     # Internal functions below
 345     def __arch_stringify(self, list):
 346         ret = []
 347         # XXX: is dist line in any config?
 348         dist = 'ac'
 349         for arch in list:
 350             ret.append(dist + '-' + arch)
 351         return ' '.join(ret)
 352
 353     def __loadpkgnames(self):
 354         def checkfiletype(name):
 355             if name[-13:]=='.src.rpm.info':
 356                 return True
 357             else:
 358                 return False
 359         list = filter(checkfiletype, os.listdir(self.basedir+'/SRPMS/.metadata'))
 360         self.pkgnames = map((lambda x: x[:-13]), list)
 361
 362     def __mark4something(self, wannabepkgs, markfunction):
 363         def chopoffextension(pkg):
 364             found = pkg.find('.src.rpm')
 365             if found == -1:
 366                 return pkg
 367             else:
 368                 return pkg[:found]
 369
 370         for wannabepkg in wannabepkgs:
 371             pkgname = chopoffextension(wannabepkg)
 372             if pkgname in self.pkgnames:
 373                 if not pkgname in self.loadedpkgs.keys():
 374                     self.loadedpkgs[pkgname]=Pkg(pkgname, self)
 375                 markfunction(self.loadedpkgs[pkgname])
 376             else:
 377                 perror('%s not found in source tree' % pkgname)
 378         bailoutonerror()
 379
 380     def __checkbuild(self, marked):
 381         """
 382         Checks queue file if all arches are built
 383
 384         Reads config.builderqueue to grab the info
 385         """
 386         f = urllib.urlopen(config.builderqueue)
 387         requests = {}
 388         reid = re.compile(r'^.*id=(.*) pri.*$')
 389         regb = re.compile(r'^group:.*$|builders:.*$', re.M)
 390         for i in re.findall(regb, f.read()):
 391             if i[0] == 'g':
 392                 id = reid.sub(r'\1', i)
 393                 requests[id] = ""
 394             elif i[0]=='b':
 395                 requests[id] = requests[id] + i
 396         f.close()
 397
 398         for pkg in marked:
 399             for bid in pkg.build.keys():
 400                 if requests.has_key(bid) and not requests[bid].find('?') == -1:
 401                     pkg.error("(buildid %s) building not finished" % bid)
 402
 403     def __checkarchs(self, dsttree, marked):
 404         """
 405         Checks marked pkgs it is built on all archs.
 406         """
 407         for pkg in marked:
 408             if len(pkg.files.keys()) <= 1:
 409                 pkg.error('has only src.rpm built')
 410                 continue
 411             otherpkgnames = self.__find_other_pkgs(pkg, dsttree)
 412
 413             # check if we're not removing some archs
 414             if otherpkgnames:
 415                 curarchs = []
 416                 missingarchs = []
 417                 for somepkg in otherpkgnames:
 418                     curarchs.extend(Pkg(somepkg, dsttree).files.keys())
 419                 for arch in curarchs:
 420                     if arch not in pkg.files.keys():
 421                         missingarchs.append(arch)
 422                 if missingarchs:
 423                     pkg.error('moving would remove archs: %s' % self.__arch_stringify(missingarchs))
 424             else:
 425                 # warn if a package isn't built for all archs
 426                 if (config.separate_noarch and 'noarch' in pkg.files.keys() and len(pkg.files.keys()) == 2):
 427                     continue
 428                 elif len(pkg.files.keys()) != len(config.ftp_archs) + 1:
 429                     missingarchs = []
 430                     for arch in config.ftp_archs:
 431                         if arch not in pkg.files.keys():
 432                             missingarchs.append(arch)
 433                     pkg.warning('not built for archs: %s' % self.__arch_stringify(missingarchs))
 434
 435     def __rmolderfromsrc(self, test = False):
 436         for pkg in self.marked4moving:
 437             olderpkgnames = self.__find_older_pkgs(pkg)
 438             for i in olderpkgnames:
 439                 Pkg(i, self).remove(test)
 440
 441     def __rmotherfromdst(self, dsttree, test = False, archivetree = None):
 442         for pkg in self.marked4moving:
 443             pkgnames = self.__find_other_pkgs(pkg, dsttree)
 444             for i in pkgnames:
 445                 if archivetree == None:
 446                     Pkg(i, dsttree).remove(test)
 447                 else:
 448                     Pkg(i, dsttree).move(archivetree, test = test)
 449
 450     # Used more than once filter functions
 451     def __find_other_pkgs(self, pkg, tree):
 452         escapedpkgname = pkg.name.replace('.', '\.').replace('+', '\+')
 453         ziewre = re.compile(escapedpkgname + '-[^-]*-[^-]*$')
 454         def filter_other_pkgs(x):
 455             if ziewre.match(x) and not x == pkg.nvr:
 456                 return True
 457             else:
 458                 return False
 459         return filter(filter_other_pkgs, tree.pkgnames)
 460
 461     def __find_older_pkgs(self, pkg):
 462         def filter_older_pkgs(x):
 463             c = x.split('-')
 464             rc = rpm.labelCompare(('0', pkg.version, pkg.release),
 465                                                         ('0', c[-2], c[-1]))
 466             if rc == 1: # pkg > x
 467                 return True
 468             else:
 469                 return False
 470         return filter(filter_older_pkgs, self.__find_other_pkgs(pkg, self))
 471
 472     def __checksigns(self, tree, pkgs, test = False):
 473         """
 474         Checks if pkgs in tree are all signed.
 475
 476         in case of test = true, error flag is set for unsigned packages
 477         """
 478         if not tree.treename in config.signed_trees:
 479             return
 480
 481         for pkg in pkgs:
 482             unsigned = 0
 483             for file in pkg.rpmfiles():
 484                 if not is_signed(file):
 485                     unsigned += 1
 486
 487             if unsigned != 0:
 488                 if test == True:
 489                     if not quietmode:
 490                         pkg.warning('%d files not signed' % unsigned)
 491                 else:
 492                     pkg.error('%d files not signed' % unsigned)
 493
 494     def __checkforobsoletes(self, tree, pkgs, test = False):
 495         """
 496         Checks queue file if package obsoletes something in destination tree and suggest for removal.
 497
 498         Only NAME tag is compared, i.e virtual packages do not get reported.
 499
 500         """
 501         if test != True:
 502             return
 503
 504         def findbyname(name):
 505             def x(nvr):
 506                 return '-'.join(nvr.split('-')[:-2]) == name
 507             return filter(x, tree.pkgnames)
 508
 509         for pkg in pkgs:
 510             obsoletes = pkg.obsoletes()
 511             if not obsoletes:
 512                 continue
 513
 514             for pn, setlist in obsoletes.items():
 515                 for item in setlist:
 516                     p = findbyname(item)
 517                     if p:
 518                         pkg.warning('obsoletes %s (via %s) in dest tree, perhaps you want rmpkg' % (p,pn))
 519
 520     def __checkforrelease(self, tree, pkgs, test = False):
 521         """
 522         Checks queue file if package release is non integer.
 523
 524         """
 525         if test != True:
 526             return
 527
 528         for pkg in pkgs:
 529             if not pkg.is_release():
 530                 pkg.warning('non-integer release: %s' % pkg.release)