modules/ftptree.py

   1 # vi: encoding=utf-8 ts=8 sts=4 sw=4 et
   2
   3 import os, config, string, urllib, re, rpm
   4 from common import fileexists, noarchcachedir
   5 from baseftptree import BasePkg, BaseFtpTree
   6 from sign import is_signed
   7
   8 errnum = 0
   9 quietmode = False
  10
  11 class SomeError(Exception):
  12     def __init__(self):
  13         return
  14
  15     def __str__(self):
  16         print "","An Error occured!"
  17
  18 def bailoutonerror():
  19     if not errnum == 0:
  20         print "%d error(s) encountered... aborting" % errnum
  21         raise SomeError
  22
  23 def pinfo(msg):
  24     print 'INFO: ' + msg
  25
  26 def perror(msg):
  27     global errnum
  28     errnum = errnum + 1
  29     print 'ERR: ' + msg
  30
  31 def pwarning(msg):
  32     print 'WARN: ' + msg
  33
  34 def rm(file, test = False):
  35     if test:
  36         if not os.path.exists(file):
  37             pinfo("TEST os.remove(%s): file doesn't exists" % file)
  38     else:
  39         try:
  40             os.remove(file)
  41         except OSError, e:
  42             pinfo("os.remove(%s): %s" % (file, e))
  43             #raise
  44
  45 def mv(src, dst, test = False):
  46     fsrc = src
  47     fdst = dst + '/' + src.split('/')[-1]
  48     if test:
  49         if not os.path.exists(src):
  50             pinfo("TEST os.rename(%s, %s): source doesn't exists" % (fsrc, fdst))
  51         if not os.path.exists(dst):
  52             pinfo("TEST destination doesn't exist: %s" % dst)
  53     else:
  54         try:
  55             os.rename(fsrc, fdst)
  56         except OSError, e:
  57             pinfo("os.rename(%s, %s): %s" % (fsrc, fdst, e))
  58             raise
  59
  60 class Pkg(BasePkg):
  61     def __init__(self, nvr, tree):
  62         BasePkg.__init__(self, nvr, tree)
  63         self.name = string.join(nvr.split('-')[:-2], '-')
  64         self.version = nvr.split('-')[-2]
  65         self.release = nvr.split('-')[-1]
  66         self.marked4removal = False
  67         self.marked4moving = False
  68         self.marked4movingpool = []
  69         self.errors = []
  70         self.warnings = []
  71
  72     def __cmp__(self, pkg):
  73         if self.name > pkg.name:
  74             return 1
  75         elif self.name < pkg.name:
  76             return -1
  77         else:
  78             return rpm.labelCompare(('0', self.version, self.release),
  79                                     ('0', pkg.version, pkg.release))
  80
  81
  82     # unfortunately can't do new Pkg(NVR), and have no "tree" in this pkg context
  83     # so this static function
  84     def is_debuginfo(self, nvr):
  85         """
  86         returns true if NVR is debuginfo package and separate debuginfo is enabled
  87         """
  88         if not config.separate_debuginfo:
  89             return False
  90         pkg = nvr.split('-')[:-2]
  91         return pkg[-1] == 'debuginfo'
  92
  93     def mark4moving(self):
  94         if not self.marked4moving:
  95             # Only one pkg in this pool can be marked for moving
  96             for pkg in self.marked4movingpool:
  97                 pkg.unmark4moving()
  98             self.tree.marked4moving.append(self)
  99             self.marked4moving=True
 100
 101     def unmark4moving(self):
 102         if self.marked4moving:
 103             self.tree.marked4moving.remove(self)
 104             self.marked4moving=False
 105
 106     def mark4removal(self):
 107         if not self.marked4removal:
 108             self.tree.marked4removal.append(self)
 109             self.marked4removal=True
 110
 111     def error(self, msg):
 112         self.errors.append(msg)
 113         if not quietmode:
 114             perror('%s %s' % (self.nvr, msg))
 115
 116     def warning(self, msg):
 117         self.warnings.append(msg)
 118         if not quietmode:
 119             pwarning('%s %s' % (self.nvr, msg))
 120
 121     def load(self, content=None):
 122         BasePkg.load(self, content)
 123         if self.info.has_key('move'):
 124             self.mark4moving()
 125
 126     def writeinfo(self):
 127         f = open(self.tree.basedir+'/SRPMS/.metadata/'+self.nvr+'.src.rpm.info', 'w')
 128         for bid in self.build.keys():
 129             f.write("info:build:%s:requester:%s\ninfo:build:%s:requester_email:%s\n" % (bid, self.build[bid].requester, bid, self.build[bid].requester_email))
 130         for key in self.info.keys():
 131             f.write("info:%s:%s\n" % (key, string.join(self.info[key], ':')))
 132         for arch in self.files.keys():
 133             for rpm in self.files[arch]:
 134                 f.write("file:%s:%s\n" % (arch, rpm))
 135
 136     def remove(self, test = False):
 137         """
 138         Remove package from ftp
 139         """
 140         for arch in self.files.keys():
 141             for rpm in self.files[arch]:
 142                 if self.is_debuginfo(rpm):
 143                     rm(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, test)
 144                 else:
 145                     rm(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, test)
 146                 if arch == 'noarch':
 147                     if fileexists(noarchcachedir + rpm + '.filelist'):
 148                         rm(noarchcachedir + rpm + '.filelist', test)
 149                     if fileexists(noarchcachedir + rpm + '.reqlist'):
 150                         rm(noarchcachedir + rpm + '.reqlist', test)
 151         rm(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', test)
 152
 153     def rpmfiles(self):
 154         """
 155         Return rpm files related to this package
 156         """
 157         files = []
 158         for arch, rpms in self.files.items():
 159             for nvr in rpms:
 160                 if self.is_debuginfo(nvr):
 161                     files.append(self.tree.basedir + '/' + arch + '/debuginfo/' + nvr)
 162                 else:
 163                     files.append(self.tree.basedir + '/' + arch + '/RPMS/' + nvr)
 164         return files
 165
 166     def move(self, dsttree, test = False):
 167         if dsttree.has_key(self.nvr):
 168             movedany = False
 169             for arch in self.files.keys():
 170                 if arch in dsttree[self.nvr].files.keys():
 171                     msg = ""
 172                     if test:
 173                         msg = "TEST "
 174                     pinfo("%sArch %s for %s is already present in dest tree; removing from srctree" % (msg, arch, self.nvr))
 175                     for rpm in self.files[arch]:
 176                         if self.is_debuginfo(rpm):
 177                             rm(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, test)
 178                         else:
 179                             rm(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, test)
 180                 else:
 181                     movedany = True
 182                     dsttree[self.nvr].files[arch] = self.files[arch]
 183                     for rpm in self.files[arch]:
 184                         if self.is_debuginfo(rpm):
 185                             mv(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, dsttree.basedir + '/' + arch + '/debuginfo/', test)
 186                         else:
 187                             mv(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, dsttree.basedir + '/' + arch + '/RPMS/', test)
 188             if not test and movedany:
 189                 for bid in self.build.keys():
 190                     dsttree[self.nvr].build[bid] = self.build[bid]
 191                 dsttree[self.nvr].writeinfo()
 192             rm(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', test)
 193         else:
 194             # move files
 195             for arch in self.files.keys():
 196                 for rpm in self.files[arch]:
 197                     if self.is_debuginfo(rpm):
 198                         mv(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, dsttree.basedir + '/' + arch + '/debuginfo/', test)
 199                     else:
 200                         mv(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, dsttree.basedir + '/' + arch + '/RPMS/', test)
 201
 202             # move metadata
 203             mv(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', dsttree.basedir + '/SRPMS/.metadata/', test)
 204
 205 class FtpTree(BaseFtpTree):
 206     def __init__(self, tree, loadall=False):
 207         BaseFtpTree.__init__(self, tree)
 208         self.loadedpkgs = {}
 209         self.marked4removal = []
 210         self.marked4moving = []
 211         self.pkgnames = []
 212         self.__loadpkgnames()
 213         if loadall:
 214             for pkgname in self.pkgnames:
 215                 self.loadedpkgs[pkgname] = Pkg(pkgname, self)
 216         # Tests:
 217         self.do_checkbuild = True
 218
 219     def __getitem__(self, key):
 220         if self.loadedpkgs.has_key(key):
 221             return self.loadedpkgs[key]
 222         elif key in self.pkgnames:
 223             pkg=Pkg(key, self)
 224             self.loadedpkgs[key]=pkg
 225             return pkg
 226         else:
 227             raise KeyError, key
 228
 229     def has_key(self, key):
 230         if key in self.pkgnames:
 231             return True
 232         else:
 233             return False
 234
 235     def keys(self):
 236         return self.pkgnames
 237
 238     def values(self):
 239         return self.loadedpkgs.values()
 240
 241     def checktree(self, dsttree):
 242         self.__checkbuild(self.loadedpkgs.values())
 243         self.__checkarchs(dsttree, self.loadedpkgs.values())
 244
 245     def testmove(self, dsttree, archivetree = None):
 246         self.__checkbuild(self.marked4moving)
 247         self.__checkarchs(dsttree, self.marked4moving)
 248
 249         self.__checksigns(dsttree, self.marked4moving, test = True)
 250
 251         self.__rmolderfromsrc(test = True)
 252         self.__rmotherfromdst(dsttree, test = True, archivetree = archivetree)
 253
 254         for pkg in self.marked4moving:
 255             pkg.move(dsttree, test = True)
 256
 257     def movepkgs(self, dsttree, archivetree = None):
 258         if self.do_checkbuild:
 259             self.__checkbuild(self.marked4moving)
 260         bailoutonerror()
 261
 262         self.__checkarchs(dsttree, self.marked4moving)
 263         bailoutonerror()
 264
 265         self.__checksigns(dsttree, self.marked4moving)
 266         bailoutonerror()
 267
 268         self.__rmolderfromsrc()
 269         self.__rmotherfromdst(dsttree, archivetree = archivetree)
 270
 271         for pkg in self.marked4moving:
 272             pkg.move(dsttree)
 273
 274     def rpmfiles(self):
 275         if self.do_checkbuild:
 276             self.__checkbuild(self.marked4moving)
 277
 278         files = []
 279         for pkg in self.marked4moving:
 280             files += pkg.rpmfiles()
 281         return files
 282
 283     def removepkgs(self):
 284         if self.do_checkbuild:
 285             self.__checkbuild(self.marked4removal)
 286         bailoutonerror()
 287         for pkg in self.marked4removal:
 288             pkg.remove()
 289
 290     def mark4removal(self, wannabepkgs):
 291         self.__mark4something(wannabepkgs, Pkg.mark4removal)
 292
 293     def mark4moving(self, wannabepkgs):
 294         self.__mark4something(wannabepkgs, Pkg.mark4moving)
 295
 296
 297     # Internal functions below
 298
 299     def __loadpkgnames(self):
 300         def checkfiletype(name):
 301             if name[-13:]=='.src.rpm.info':
 302                 return True
 303             else:
 304                 return False
 305         list = filter(checkfiletype, os.listdir(self.basedir+'/SRPMS/.metadata'))
 306         self.pkgnames = map((lambda x: x[:-13]), list)
 307
 308     def __mark4something(self, wannabepkgs, markfunction):
 309         def chopoffextension(pkg):
 310             found=pkg.find('.src.rpm')
 311             if found==-1:
 312                 return pkg
 313             else:
 314                 return pkg[:found]
 315         for wannabepkg in wannabepkgs:
 316             pkgname=chopoffextension(wannabepkg)
 317             if pkgname in self.pkgnames:
 318                 if not pkgname in self.loadedpkgs.keys():
 319                     self.loadedpkgs[pkgname]=Pkg(pkgname, self)
 320                 markfunction(self.loadedpkgs[pkgname])
 321             else:
 322                 perror('%s not found in source tree' % pkgname)
 323         bailoutonerror()
 324
 325     def __checkbuild(self, marked):
 326         """
 327         Checks queue file if all arches are built
 328
 329         Reads config.builderqueue to grab the info
 330         """
 331         f = urllib.urlopen(config.builderqueue)
 332         requests = {}
 333         reid = re.compile(r'^.*id=(.*) pri.*$')
 334         regb = re.compile(r'^group:.*$|builders:.*$', re.M)
 335         for i in re.findall(regb, f.read()):
 336             if i[0] == 'g':
 337                 id = reid.sub(r'\1', i)
 338                 requests[id] = ""
 339             elif i[0]=='b':
 340                 requests[id] = requests[id] + i
 341         f.close()
 342
 343         for pkg in marked:
 344             for bid in pkg.build.keys():
 345                 if requests.has_key(bid) and not requests[bid].find('?') == -1:
 346                     pkg.error("(buildid %s) building not finished" % bid)
 347
 348     def __checkarchs(self, dsttree, marked):
 349         """
 350         Checks marked pkgs it is built on all archs.
 351         """
 352         for pkg in marked:
 353             if len(pkg.files.keys()) <= 1:
 354                 pkg.error('has only src.rpm built')
 355                 continue
 356             otherpkgnames = self.__find_other_pkgs(pkg, dsttree)
 357
 358             # check if we're not removing some archs
 359             if otherpkgnames:
 360                 curarchs = []
 361                 missingarchs = []
 362                 for somepkg in otherpkgnames:
 363                     curarchs.extend(Pkg(somepkg, dsttree).files.keys())
 364                 for arch in curarchs:
 365                     if arch not in pkg.files.keys():
 366                         missingarchs.append(arch)
 367                 if missingarchs:
 368                     pkg.error('moving would remove archs: %s' % missingarchs)
 369             else:
 370                 # warn if a package isn't built for all archs
 371                 if (config.separate_noarch and 'noarch' in pkg.files.keys() and len(pkg.files.keys()) == 2):
 372                     continue
 373                 elif len(pkg.files.keys()) != len(config.ftp_archs) + 1:
 374                     missingarchs = []
 375                     for arch in config.ftp_archs:
 376                         if arch not in pkg.files.keys():
 377                             missingarchs.append(arch)
 378                     pkg.warning('not built for archs: %s' % missingarchs)
 379
 380     def __rmolderfromsrc(self, test = False):
 381         for pkg in self.marked4moving:
 382             olderpkgnames = self.__find_older_pkgs(pkg)
 383             for i in olderpkgnames:
 384                 Pkg(i, self).remove(test)
 385
 386     def __rmotherfromdst(self, dsttree, test = False, archivetree = None):
 387         for pkg in self.marked4moving:
 388             pkgnames = self.__find_other_pkgs(pkg, dsttree)
 389             for i in pkgnames:
 390                 if archivetree == None:
 391                     Pkg(i, dsttree).remove(test)
 392                 else:
 393                     Pkg(i, dsttree).move(archivetree, test = test)
 394
 395     # Used more than once filter functions
 396     def __find_other_pkgs(self, pkg, tree):
 397         escapedpkgname = pkg.name.replace('.', '\.').replace('+', '\+')
 398         ziewre = re.compile(escapedpkgname + '-[^-]*-[^-]*$')
 399         def filter_other_pkgs(x):
 400             if ziewre.match(x) and not x == pkg.nvr:
 401                 return True
 402             else:
 403                 return False
 404         return filter(filter_other_pkgs, tree.pkgnames)
 405
 406     def __find_older_pkgs(self, pkg):
 407         def filter_older_pkgs(x):
 408             c = x.split('-')
 409             rc = rpm.labelCompare(('0', pkg.version, pkg.release),
 410                                                         ('0', c[-2], c[-1]))
 411             if rc == 1: # pkg > x
 412                 return True
 413             else:
 414                 return False
 415         return filter(filter_older_pkgs, self.__find_other_pkgs(pkg, self))
 416
 417     def __checksigns(self, tree, pkgs, test = False):
 418         """
 419         Checks if pkgs in tree are all signed.
 420
 421         in case of test = true, error flag is set for unsigned packages
 422         """
 423         if not tree.treename in config.signed_trees:
 424             return
 425
 426         for pkg in pkgs:
 427             unsigned = 0
 428             for file in pkg.rpmfiles():
 429                 if not is_signed(file):
 430                     unsigned += 1;
 431
 432             if unsigned != 0:
 433                 if test == True:
 434                     if not quietmode:
 435                         pkg.warning('%d files not signed' % unsigned)
 436                 else:
 437                     pkg.error('%d files not signed' % unsigned)