modules/ftptree.py

   1 # vi: encoding=utf-8 ts=8 sts=4 sw=4 et
   2
   3 from __future__ import print_function
   4
   5 import os
   6 import config
   7 import string
   8 try:
   9     import urllib.request as urlmess
  10 except ImportError:
  11     import urllib as urlmess
  12     pass
  13 import re
  14 import rpm
  15 from common import fileexists, noarchcachedir
  16 from baseftptree import BasePkg, BaseFtpTree
  17 from sign import is_signed
  18
  19 errnum = 0
  20 quietmode = False
  21
  22 class SomeError(Exception):
  23     def __init__(self):
  24         return
  25
  26     def __str__(self):
  27         return "An Error occured!"
  28
  29 def bailoutonerror():
  30     if not errnum == 0:
  31         print("%d error(s) encountered... aborting" % errnum)
  32         raise SomeError()
  33
  34 def pinfo(msg):
  35     print('INFO: ' + msg)
  36
  37 def perror(msg):
  38     global errnum
  39     errnum = errnum + 1
  40     print('ERR: ' + msg)
  41
  42 def pwarning(msg):
  43     print('WARN: ' + msg)
  44
  45 def rm(file, test = False):
  46     if test:
  47         if not os.path.exists(file):
  48             pinfo("TEST os.remove(%s): file doesn't exists" % file)
  49     else:
  50         try:
  51             os.remove(file)
  52         except OSError as e:
  53             pinfo("os.remove(%s): %s" % (file, e))
  54             #raise
  55
  56 def mv(src, dst, test = False):
  57     fsrc = src
  58     fdst = dst + '/' + src.split('/')[-1]
  59     if test:
  60         if not os.path.exists(fsrc):
  61             pinfo("TEST os.rename(%s, %s): source doesn't exists" % (fsrc, fdst))
  62         if not os.path.exists(dst):
  63             pinfo("TEST destination doesn't exist: %s" % dst)
  64     else:
  65         try:
  66             os.rename(fsrc, fdst)
  67         except OSError as e:
  68             pinfo("os.rename(%s, %s): %s" % (fsrc, fdst, e))
  69             raise
  70
  71 class Pkg(BasePkg):
  72     def __init__(self, nvr, tree):
  73         BasePkg.__init__(self, nvr, tree)
  74         self.name = '-'.join(nvr.split('-')[:-2])
  75         self.version = nvr.split('-')[-2]
  76         self.release = nvr.split('-')[-1]
  77         self.marked4removal = False
  78         self.marked4moving = False
  79         self.marked4movingpool = []
  80         self.errors = []
  81         self.warnings = []
  82
  83     def __cmp__(self, pkg):
  84         if self.name > pkg.name:
  85             return 1
  86         elif self.name < pkg.name:
  87             return -1
  88         else:
  89             return rpm.labelCompare(('0', self.version, self.release),
  90                                     ('0', pkg.version, pkg.release))
  91
  92
  93     # unfortunately can't do new Pkg(NVR), and have no "tree" in this pkg context
  94     # so this static function
  95     def is_debuginfo(self, nvr):
  96         """
  97         returns true if NVR is debuginfo package and separate debuginfo is enabled
  98         """
  99         if not config.separate_debuginfo:
 100             return False
 101         pkg = nvr.split('-')[:-2]
 102         return pkg[-1] == 'debuginfo'
 103
 104     def is_sourcefile(self, file):
 105         """
 106         returns true if file is source package
 107         """
 108         return file[-8:] == '.src.rpm'
 109
 110     # returns true if package build is integer
 111     def is_release(self):
 112         """
 113         To account Release tags with subver macros, we consider integer release
 114         if it contains odd number of dots:
 115
 116         1 -> True
 117         0.1 -> False
 118         0.%{subver}.%{rel}, %{rel} = 1 -> 0.20010.1 -> True
 119         0.%{subver}.%{rel}, %{rel} = 0.1 -> 0.20010.0.1 -> False
 120         """
 121         return self.release.count('.') % 2 == 0
 122
 123     def mark4moving(self):
 124         if not self.marked4moving:
 125             # Only one pkg in this pool can be marked for moving
 126             for pkg in self.marked4movingpool:
 127                 pkg.unmark4moving()
 128             self.tree.marked4moving.append(self)
 129             self.marked4moving=True
 130
 131     def unmark4moving(self):
 132         if self.marked4moving:
 133             self.tree.marked4moving.remove(self)
 134             self.marked4moving=False
 135
 136     def mark4removal(self):
 137         if not self.marked4removal:
 138             self.tree.marked4removal.append(self)
 139             self.marked4removal=True
 140
 141     def error(self, msg):
 142         self.errors.append(msg)
 143         if not quietmode:
 144             perror('%s %s' % (self.nvr, msg))
 145
 146     def warning(self, msg):
 147         self.warnings.append(msg)
 148         if not quietmode:
 149             pwarning('%s %s' % (self.nvr, msg))
 150
 151     def load(self, content=None):
 152         BasePkg.load(self, content)
 153         if 'move' in self.info:
 154             self.mark4moving()
 155
 156     def writeinfo(self):
 157         f = open(self.tree.basedir+'/SRPMS/.metadata/'+self.nvr+'.src.rpm.info', 'w')
 158         for bid in self.build.keys():
 159             f.write("info:build:%s:requester:%s\ninfo:build:%s:requester_email:%s\n" % (bid, self.build[bid].requester, bid, self.build[bid].requester_email))
 160         for key in self.info.keys():
 161             f.write("info:%s:%s\n" % (key, ':'.join(self.info[key])))
 162         for arch in self.files.keys():
 163             for rpm in self.files[arch]:
 164                 f.write("file:%s:%s\n" % (arch, rpm))
 165
 166     def remove(self, test = False):
 167         """
 168         Remove package from ftp
 169         """
 170         for arch in self.files.keys():
 171             for rpm in self.files[arch]:
 172                 if self.is_debuginfo(rpm):
 173                     rm(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, test)
 174                 else:
 175                     rm(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, test)
 176                 if arch == 'noarch':
 177                     if fileexists(noarchcachedir + rpm + '.filelist'):
 178                         rm(noarchcachedir + rpm + '.filelist', test)
 179                     if fileexists(noarchcachedir + rpm + '.reqlist'):
 180                         rm(noarchcachedir + rpm + '.reqlist', test)
 181         rm(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', test)
 182
 183     def rpmfiles(self, debugfiles = True, sourcefiles  = True):
 184         """
 185         Return rpm files related to this package
 186         """
 187         files = []
 188         for arch, rpms in self.files.items():
 189             for nvr in rpms:
 190                 if self.is_debuginfo(nvr):
 191                     if debugfiles:
 192                         files.append(self.tree.basedir + '/' + arch + '/debuginfo/' + nvr)
 193                 else:
 194                     if self.is_sourcefile(nvr):
 195                         if sourcefiles:
 196                             files.append(self.tree.basedir + '/' + arch + '/RPMS/' + nvr)
 197                     else:
 198                         files.append(self.tree.basedir + '/' + arch + '/RPMS/' + nvr)
 199         return files
 200
 201     def obsoletes(self):
 202         """
 203         Return obsoletes for all packages in Pkg:
 204
 205         {'php-geshi': set(['geshi'])}
 206
 207         """
 208         def rpmhdr(pkg):
 209             ts = rpm.ts()
 210             ts.setVSFlags(rpm.RPMVSF_NODSAHEADER)
 211             fdno = os.open(pkg, os.O_RDONLY)
 212             hdr = ts.hdrFromFdno(fdno)
 213             os.close(fdno)
 214             return hdr
 215
 216         obsoletes = {}
 217         for rpmfile in self.rpmfiles():
 218             if not os.path.exists(rpmfile):
 219                 continue
 220             hdr = rpmhdr(rpmfile)
 221             if not hdr[rpm.RPMTAG_OBSOLETES]:
 222                 continue
 223
 224             name = hdr[rpm.RPMTAG_NAME]
 225             if not name in obsoletes:
 226                 obsoletes[name] = set()
 227
 228             for tag in hdr[rpm.RPMTAG_OBSOLETES]:
 229                 obsoletes[name].add(tag)
 230
 231         return obsoletes
 232
 233     def move(self, dsttree, test = False):
 234         if dsttree.has_key(self.nvr):
 235             movedany = False
 236             for arch in self.files.keys():
 237                 if arch in dsttree[self.nvr].files.keys():
 238                     msg = ""
 239                     if test:
 240                         msg = "TEST "
 241                     pinfo("%sArch %s for %s is already present in dest tree; removing from srctree" % (msg, arch, self.nvr))
 242                     for rpm in self.files[arch]:
 243                         if self.is_debuginfo(rpm):
 244                             rm(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, test)
 245                         else:
 246                             rm(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, test)
 247                 else:
 248                     movedany = True
 249                     dsttree[self.nvr].files[arch] = self.files[arch]
 250                     for rpm in self.files[arch]:
 251                         if self.is_debuginfo(rpm):
 252                             mv(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, dsttree.basedir + '/' + arch + '/debuginfo/', test)
 253                         else:
 254                             mv(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, dsttree.basedir + '/' + arch + '/RPMS/', test)
 255             if not test and movedany:
 256                 for bid in self.build.keys():
 257                     dsttree[self.nvr].build[bid] = self.build[bid]
 258                 dsttree[self.nvr].writeinfo()
 259             rm(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', test)
 260         else:
 261             # move files
 262             for arch in self.files.keys():
 263                 for rpm in self.files[arch]:
 264                     if self.is_debuginfo(rpm):
 265                         mv(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, dsttree.basedir + '/' + arch + '/debuginfo/', test)
 266                     else:
 267                         mv(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, dsttree.basedir + '/' + arch + '/RPMS/', test)
 268
 269             # move metadata
 270             mv(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', dsttree.basedir + '/SRPMS/.metadata/', test)
 271
 272 class FtpTree(BaseFtpTree):
 273     def __init__(self, tree, loadall=False):
 274         BaseFtpTree.__init__(self, tree)
 275         self.loadedpkgs = {}
 276         self.marked4removal = []
 277         self.marked4moving = []
 278         self.pkgnames = []
 279         self.__loadpkgnames()
 280         if loadall:
 281             for pkgname in self.pkgnames:
 282                 self.loadedpkgs[pkgname] = Pkg(pkgname, self)
 283         # Tests:
 284         self.do_checkbuild = True
 285
 286     def __getitem__(self, key):
 287         if key in self.loadedpkgs:
 288             return self.loadedpkgs[key]
 289         elif key in self.pkgnames:
 290             pkg=Pkg(key, self)
 291             self.loadedpkgs[key]=pkg
 292             return pkg
 293         else:
 294             raise KeyError(key)
 295
 296     def has_key(self, key):
 297         if key in self.pkgnames:
 298             return True
 299         else:
 300             return False
 301
 302     def keys(self):
 303         return self.pkgnames
 304
 305     def values(self):
 306         return self.loadedpkgs.values()
 307
 308     def checktree(self, dsttree):
 309         self.__checkbuild(self.loadedpkgs.values())
 310         self.__checkarchs(dsttree, self.loadedpkgs.values())
 311
 312     def testmove(self, dsttree, archivetree = None):
 313         self.__checkbuild(self.marked4moving)
 314         self.__checkarchs(dsttree, self.marked4moving)
 315         if not dsttree.treename.count("archive"):
 316             self.__checkduplicates(self.marked4moving)
 317
 318         self.__checksigns(dsttree, self.marked4moving, test = True)
 319         self.__checkforobsoletes(dsttree, self.marked4moving, test = True)
 320         self.__checkforrelease(dsttree, self.marked4moving, test = True)
 321
 322         if not self.treename.count("archive"):
 323             self.__rmolderfromsrc(test = True)
 324         if not dsttree.treename.count("archive"):
 325             self.__rmotherfromdst(dsttree, test = True, archivetree = archivetree)
 326
 327         for pkg in self.marked4moving:
 328             pkg.move(dsttree, test = True)
 329
 330     def movepkgs(self, dsttree, archivetree = None):
 331         if self.do_checkbuild:
 332             self.__checkbuild(self.marked4moving)
 333         bailoutonerror()
 334
 335         self.__checkarchs(dsttree, self.marked4moving)
 336         bailoutonerror()
 337
 338         self.__checksigns(dsttree, self.marked4moving)
 339         bailoutonerror()
 340
 341         if not self.treename.count("archive"):
 342             self.__rmolderfromsrc()
 343         if not dsttree.treename.count("archive"):
 344             self.__rmotherfromdst(dsttree, archivetree = archivetree)
 345
 346         for pkg in self.marked4moving:
 347             pkg.move(dsttree)
 348
 349     def rpmfiles(self, debugfiles = True, sourcefiles = True):
 350         if self.do_checkbuild:
 351             self.__checkbuild(self.marked4moving)
 352
 353         files = []
 354         for pkg in self.marked4moving:
 355             files += pkg.rpmfiles(debugfiles = debugfiles, sourcefiles = sourcefiles)
 356         return files
 357
 358     def removepkgs(self):
 359         if self.do_checkbuild:
 360             self.__checkbuild(self.marked4removal)
 361         bailoutonerror()
 362         for pkg in self.marked4removal:
 363             pkg.remove()
 364
 365     def mark4removal(self, wannabepkgs):
 366         self.__mark4something(wannabepkgs, Pkg.mark4removal)
 367
 368     def mark4moving(self, wannabepkgs):
 369         self.__mark4something(wannabepkgs, Pkg.mark4moving)
 370
 371     # Internal functions below
 372     def __arch_stringify(self, list):
 373         ret = []
 374         dist = config.ftp_dist;
 375         for arch in list:
 376             ret.append(dist + '-' + arch)
 377         return ' '.join(ret)
 378
 379     def __loadpkgnames(self):
 380         def checkfiletype(name):
 381             if name[-13:]=='.src.rpm.info':
 382                 return True
 383             else:
 384                 return False
 385         pkglist = list(filter(checkfiletype, os.listdir(self.basedir+'/SRPMS/.metadata')))
 386         self.pkgnames = list(map((lambda x: x[:-13]), pkglist))
 387
 388     def __mark4something(self, wannabepkgs, markfunction):
 389         def chopoffextension(pkg):
 390             found = pkg.find('.src.rpm')
 391             if found == -1:
 392                 return pkg
 393             else:
 394                 return pkg[:found]
 395
 396         for wannabepkg in wannabepkgs:
 397             pkgname = chopoffextension(wannabepkg)
 398             if pkgname in self.pkgnames:
 399                 if not pkgname in self.loadedpkgs.keys():
 400                     self.loadedpkgs[pkgname]=Pkg(pkgname, self)
 401                 markfunction(self.loadedpkgs[pkgname])
 402             else:
 403                 perror('%s not found in source tree' % pkgname)
 404         bailoutonerror()
 405
 406     def __checkbuild(self, marked):
 407         """
 408         Checks queue file if all arches are built
 409
 410         Reads config.builderqueue to grab the info
 411         """
 412         f = urlmess.urlopen(config.builderqueue)
 413         requests = {}
 414         reid = re.compile(r'^.*id=(.*) pri.*$')
 415         regb = re.compile(r'^group:.*$|builders:.*$', re.M)
 416         for i in re.findall(regb, f.read().decode('utf-8')):
 417             if i[0] == 'g':
 418                 id = reid.sub(r'\1', i)
 419                 requests[id] = ""
 420             elif i[0]=='b':
 421                 requests[id] = requests[id] + i
 422         f.close()
 423
 424         for pkg in marked:
 425             for bid in pkg.build.keys():
 426                 if bid in requests and not requests[bid].find('?') == -1:
 427                     pkg.error("(buildid %s) building not finished" % bid)
 428
 429     def __checkarchs(self, dsttree, marked):
 430         """
 431         Checks marked pkgs it is built on all archs.
 432         """
 433         for pkg in marked:
 434             if len(pkg.files.keys()) <= 1:
 435                 pkg.error('has only src.rpm built')
 436                 continue
 437             otherpkgnames = self.__find_other_pkgs(pkg, dsttree)
 438
 439             # check if we're not removing some archs
 440             if otherpkgnames:
 441                 curarchs = []
 442                 missingarchs = []
 443                 for somepkg in otherpkgnames:
 444                     curarchs.extend(Pkg(somepkg, dsttree).files.keys())
 445                 for arch in curarchs:
 446                     if arch not in pkg.files.keys():
 447                         missingarchs.append(arch)
 448                 if missingarchs:
 449                     pkg.error('moving would remove archs: %s' % self.__arch_stringify(missingarchs))
 450             else:
 451                 # warn if a package isn't built for all archs
 452                 # ftp_archs + SRPMS
 453                 ftp_archs_num = len(config.ftp_archs) + 1
 454                 if (config.separate_noarch and 'noarch' in pkg.files.keys()):
 455                     # ftp_archs + SRPMS + noarch subpackages
 456                     ftp_archs_num += 1
 457                     # plain simple noarch package
 458                     if (len(pkg.files.keys()) == 2):
 459                         continue
 460
 461                 if len(pkg.files.keys()) != ftp_archs_num:
 462                     missingarchs = []
 463                     for arch in config.ftp_archs:
 464                         if arch not in pkg.files.keys():
 465                             missingarchs.append(arch)
 466                     pkg.warning('not built for archs: %s' % self.__arch_stringify(missingarchs))
 467
 468     def __checkduplicates(self, marked):
 469         """
 470         Checks if marked packages contain duplicate packages (with different versions)
 471         """
 472         for pkg in marked:
 473             olderpkgnames = self.__find_older_pkgs(pkg)
 474             for i in olderpkgnames:
 475                 markednames = [str(x) for x in marked]
 476                 if i in markednames:
 477                     pkg.error('duplicate package: %s' % i)
 478
 479     def __rmolderfromsrc(self, test = False):
 480         for pkg in self.marked4moving:
 481             olderpkgnames = self.__find_older_pkgs(pkg)
 482             for i in olderpkgnames:
 483                 Pkg(i, self).remove(test)
 484
 485     def __rmotherfromdst(self, dsttree, test = False, archivetree = None):
 486         for pkg in self.marked4moving:
 487             pkgnames = self.__find_other_pkgs(pkg, dsttree)
 488             for i in pkgnames:
 489                 if archivetree == None:
 490                     Pkg(i, dsttree).remove(test)
 491                 else:
 492                     Pkg(i, dsttree).move(archivetree, test = test)
 493
 494     # Used more than once filter functions
 495     def __find_other_pkgs(self, pkg, tree):
 496         escapedpkgname = pkg.name.replace('.', '\.').replace('+', '\+')
 497         ziewre = re.compile(escapedpkgname + '-[^-]*-[^-]*$')
 498         def filter_other_pkgs(x):
 499             if ziewre.match(x) and not x == pkg.nvr:
 500                 return True
 501             else:
 502                 return False
 503         return list(filter(filter_other_pkgs, tree.pkgnames))
 504
 505     def __find_older_pkgs(self, pkg):
 506         def filter_older_pkgs(x):
 507             c = x.split('-')
 508             rc = rpm.labelCompare(('0', pkg.version, pkg.release),
 509                                                         ('0', c[-2], c[-1]))
 510             if rc == 1: # pkg > x
 511                 return True
 512             else:
 513                 return False
 514         return list(filter(filter_older_pkgs, self.__find_other_pkgs(pkg, self)))
 515
 516     def __checksigns(self, tree, pkgs, test = False):
 517         """
 518         Checks if pkgs in tree are all signed.
 519
 520         in case of test = true, error flag is set for unsigned packages
 521         """
 522         if not tree.treename in config.signed_trees:
 523             return
 524
 525         for pkg in pkgs:
 526             unsigned = 0
 527             for file in pkg.rpmfiles():
 528                 if not is_signed(file):
 529                     unsigned += 1
 530
 531             if unsigned != 0:
 532                 if test == True:
 533                     if not quietmode:
 534                         pkg.warning('%d files not signed' % unsigned)
 535                 else:
 536                     pkg.error('%d files not signed' % unsigned)
 537
 538     def __checkforobsoletes(self, tree, pkgs, test = False):
 539         """
 540         Checks queue file if package obsoletes something in destination tree and suggest for removal.
 541
 542         Only NAME tag is compared, i.e virtual packages do not get reported.
 543
 544         """
 545         if test != True:
 546             return
 547
 548         def findbyname(name):
 549             def x(nvr):
 550                 return '-'.join(nvr.split('-')[:-2]) == name
 551             return list(filter(x, tree.pkgnames))
 552
 553         for pkg in pkgs:
 554             obsoletes = pkg.obsoletes()
 555             if not obsoletes:
 556                 continue
 557
 558             for pn, setlist in obsoletes.items():
 559                 for item in setlist:
 560                     p = findbyname(item)
 561                     if p:
 562                         pkg.warning('obsoletes %s (via %s) in dest tree, perhaps you want rmpkg' % (p,pn))
 563
 564     def __checkforrelease(self, tree, pkgs, test = False):
 565         """
 566         Checks queue file if package release is non integer.
 567
 568         """
 569         if test != True:
 570             return
 571
 572         for pkg in pkgs:
 573             if not pkg.is_release():
 574                 pkg.warning('non-integer release: %s' % pkg.release)