modules/ftptree.py

   1 # vi: encoding=utf-8 ts=8 sts=4 sw=4 et
   2
   3 import os, config, string, urllib, re, rpm
   4 from common import fileexists, noarchcachedir
   5 from baseftptree import BasePkg, BaseFtpTree
   6 errnum=0
   7
   8 quietmode=False
   9
  10 class SomeError(Exception):
  11     def __init__(self):
  12         return
  13
  14     def __str__(self):
  15         print "","An Error occured!"
  16
  17 def bailoutonerror():
  18     if not errnum == 0:
  19         print "%d error(s) encountered... aborting" % errnum
  20         raise SomeError
  21
  22 def pinfo(msg):
  23     print 'INFO: ' + msg
  24
  25 def perror(msg):
  26     global errnum
  27     errnum=errnum+1
  28     print 'ERR: ' + msg
  29
  30 def pwarning(msg):
  31     print 'WARN: ' + msg
  32
  33 def rm(file, test=False):
  34     if test:
  35         if not os.path.exists(file):
  36             pinfo("TEST os.remove(%s): file doesn't exists" % file)
  37     else:
  38         try:
  39             os.remove(file)
  40         except OSError, e:
  41             pinfo("os.remove(%s): %s" % (file, e))
  42             #raise
  43
  44 def mv(src, dst, test=False):
  45     fsrc = src
  46     fdst = dst+'/'+src.split('/')[-1]
  47     if test:
  48         if not os.path.exists(src):
  49             pinfo("TEST os.rename(%s, %s): source doesn't exists" % (fsrc, fdst))
  50     else:
  51         try:
  52             os.rename(fsrc, fdst)
  53         except OSError, e:
  54             pinfo("os.rename(%s, %s): %s" % (fsrc, fdst, e))
  55             raise
  56
  57 class Pkg(BasePkg):
  58     def __init__(self, nvr, tree):
  59         BasePkg.__init__(self, nvr, tree)
  60         self.name = string.join(nvr.split('-')[:-2], '-')
  61         self.version = nvr.split('-')[-2]
  62         self.release = nvr.split('-')[-1]
  63         self.marked4removal = False
  64         self.marked4moving = False
  65         self.marked4movingpool = []
  66         self.errors = []
  67         self.warnings = []
  68
  69     def __cmp__(self, pkg):
  70         if self.name > pkg.name:
  71             return 1
  72         elif self.name < pkg.name:
  73             return -1
  74         else:
  75             return rpm.labelCompare(('0', self.version, self.release),
  76                                     ('0', pkg.version, pkg.release))
  77
  78
  79     # unfortunately can't do new Pkg(NVR), and have no "tree" in this pkg context
  80     # so this static function
  81     def is_debuginfo(self, nvr):
  82         """
  83         returns true if NVR is debuginfo package and separate debuginfo is enabled
  84         """
  85         if not config.separate_debuginfo:
  86             return False
  87         pkg = nvr.split('-')[:-2]
  88         return pkg[-1] == 'debuginfo'
  89
  90     def mark4moving(self):
  91         if not self.marked4moving:
  92             # Only one pkg in this pool can be marked for moving
  93             for pkg in self.marked4movingpool:
  94                 pkg.unmark4moving()
  95             self.tree.marked4moving.append(self)
  96             self.marked4moving=True
  97
  98     def unmark4moving(self):
  99         if self.marked4moving:
 100             self.tree.marked4moving.remove(self)
 101             self.marked4moving=False
 102
 103     def mark4removal(self):
 104         if not self.marked4removal:
 105             self.tree.marked4removal.append(self)
 106             self.marked4removal=True
 107
 108     def error(self, msg):
 109         self.errors.append(msg)
 110         if not quietmode:
 111             perror('%s %s' % (self.nvr, msg))
 112
 113     def warning(self, msg):
 114         self.warnings.append(msg)
 115         if not quietmode:
 116             pwarning('%s %s' % (self.nvr, msg))
 117
 118     def load(self, content=None):
 119         BasePkg.load(self, content)
 120         if self.info.has_key('move'):
 121             self.mark4moving()
 122
 123     def writeinfo(self):
 124         f = open(self.tree.basedir+'/SRPMS/.metadata/'+self.nvr+'.src.rpm.info', 'w')
 125         for bid in self.build.keys():
 126             f.write("info:build:%s:requester:%s\ninfo:build:%s:requester_email:%s\n" % (bid, self.build[bid].requester, bid, self.build[bid].requester_email))
 127         for key in self.info.keys():
 128             f.write("info:%s:%s\n" % (key, string.join(self.info[key], ':')))
 129         for arch in self.files.keys():
 130             for rpm in self.files[arch]:
 131                 f.write("file:%s:%s\n" % (arch, rpm))
 132
 133     def remove(self, test = False):
 134         """
 135         Remove package from ftp
 136         """
 137         for arch in self.files.keys():
 138             for rpm in self.files[arch]:
 139                 if self.is_debuginfo(rpm):
 140                     rm(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, test)
 141                 else:
 142                     rm(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, test)
 143                 if arch == 'noarch':
 144                     if fileexists(noarchcachedir + rpm + '.filelist'):
 145                         rm(noarchcachedir + rpm + '.filelist', test)
 146                     if fileexists(noarchcachedir + rpm + '.reqlist'):
 147                         rm(noarchcachedir + rpm + '.reqlist', test)
 148         rm(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', test)
 149
 150     def rpmfiles(self):
 151         """
 152         Return rpm files related to this package
 153         """
 154         files = []
 155         for arch, rpms in self.files.items():
 156             for nvr in rpms:
 157                 if self.is_debuginfo(nvr):
 158                     files.append(self.tree.basedir + '/' + arch + '/debuginfo/' + nvr)
 159                 else:
 160                     files.append(self.tree.basedir + '/' + arch + '/RPMS/' + nvr)
 161         return files
 162
 163     def move(self, dsttree, test=False):
 164         if dsttree.has_key(self.nvr):
 165             movedany = False
 166             for arch in self.files.keys():
 167                 if arch in dsttree[self.nvr].files.keys():
 168                     msg = ""
 169                     if test:
 170                         msg = "TEST "
 171                     pinfo("%sArch %s for %s is already present in dest tree; removing from srctree" % (msg, arch, self.nvr))
 172                     for rpm in self.files[arch]:
 173                         if self.is_debuginfo(rpm):
 174                             rm(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, test)
 175                         else:
 176                             rm(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, test)
 177                 else:
 178                     movedany = True
 179                     dsttree[self.nvr].files[arch] = self.files[arch]
 180                     for rpm in self.files[arch]:
 181                         if self.is_debuginfo(rpm):
 182                             mv(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, dsttree.basedir + '/' + arch + '/debuginfo/', test)
 183                         else:
 184                             mv(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, dsttree.basedir + '/' + arch + '/RPMS/', test)
 185             if not test and  movedany:
 186                 for bid in self.build.keys():
 187                     dsttree[self.nvr].build[bid] = self.build[bid]
 188                 dsttree[self.nvr].writeinfo()
 189             rm(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', test)
 190         else:
 191             # move files
 192             for arch in self.files.keys():
 193                 for rpm in self.files[arch]:
 194                     if self.is_debuginfo(rpm):
 195                         mv(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, dsttree.basedir + '/' + arch + '/debuginfo/', test)
 196                     else:
 197                         mv(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, dsttree.basedir + '/' + arch + '/RPMS/', test)
 198
 199             # move metadata
 200             mv(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', dsttree.basedir + '/SRPMS/.metadata/', test)
 201
 202 class FtpTree(BaseFtpTree):
 203     def __init__(self, tree, loadall=False):
 204         BaseFtpTree.__init__(self, tree)
 205         self.loadedpkgs = {}
 206         self.marked4removal = []
 207         self.marked4moving = []
 208         self.pkgnames = []
 209         self.__loadpkgnames()
 210         if loadall:
 211             for pkgname in self.pkgnames:
 212                 self.loadedpkgs[pkgname] = Pkg(pkgname, self)
 213         # Tests:
 214         self.do_checkbuild = True
 215
 216     def __getitem__(self, key):
 217         if self.loadedpkgs.has_key(key):
 218             return self.loadedpkgs[key]
 219         elif key in self.pkgnames:
 220             pkg=Pkg(key, self)
 221             self.loadedpkgs[key]=pkg
 222             return pkg
 223         else:
 224             raise KeyError, key
 225
 226     def has_key(self, key):
 227         if key in self.pkgnames:
 228             return True
 229         else:
 230             return False
 231
 232     def keys(self):
 233         return self.pkgnames
 234
 235     def values(self):
 236         return self.loadedpkgs.values()
 237
 238     def checktree(self, dsttree):
 239         self.__checkbuild(self.loadedpkgs.values())
 240         self.__checkarchs(dsttree, self.loadedpkgs.values())
 241
 242     def testmove(self, dsttree):
 243         self.__checkbuild(self.marked4moving)
 244         self.__checkarchs(dsttree, self.marked4moving)
 245
 246         self.__rmolderfromsrc(test = True)
 247         self.__rmotherfromdst(dsttree, test = True)
 248
 249         for pkg in self.marked4moving:
 250             pkg.move(dsttree, test = True)
 251
 252     def movepkgs(self, dsttree):
 253         if self.do_checkbuild:
 254             self.__checkbuild(self.marked4moving)
 255         bailoutonerror()
 256         self.__checkarchs(dsttree, self.marked4moving)
 257         bailoutonerror()
 258         self.__rmolderfromsrc()
 259         self.__rmotherfromdst(dsttree)
 260
 261         for pkg in self.marked4moving:
 262             pkg.move(dsttree)
 263
 264     def rpmfiles(self):
 265         if self.do_checkbuild:
 266             self.__checkbuild(self.marked4moving)
 267         bailoutonerror()
 268
 269         files = []
 270         for pkg in self.marked4moving:
 271             files += pkg.rpmfiles()
 272         return files
 273
 274     def removepkgs(self):
 275         if self.do_checkbuild:
 276             self.__checkbuild(self.marked4removal)
 277         bailoutonerror()
 278         for pkg in self.marked4removal:
 279             pkg.remove()
 280
 281     def mark4removal(self, wannabepkgs):
 282         self.__mark4something(wannabepkgs, Pkg.mark4removal)
 283
 284     def mark4moving(self, wannabepkgs):
 285         self.__mark4something(wannabepkgs, Pkg.mark4moving)
 286
 287
 288     # Internal functions below
 289
 290     def __loadpkgnames(self):
 291         def checkfiletype(name):
 292             if name[-13:]=='.src.rpm.info':
 293                 return True
 294             else:
 295                 return False
 296         list = filter(checkfiletype, os.listdir(self.basedir+'/SRPMS/.metadata'))
 297         self.pkgnames = map((lambda x: x[:-13]), list)
 298
 299     def __mark4something(self, wannabepkgs, markfunction):
 300         def chopoffextension(pkg):
 301             found=pkg.find('.src.rpm')
 302             if found==-1:
 303                 return pkg
 304             else:
 305                 return pkg[:found]
 306         for wannabepkg in wannabepkgs:
 307             pkgname=chopoffextension(wannabepkg)
 308             if pkgname in self.pkgnames:
 309                 if not pkgname in self.loadedpkgs.keys():
 310                     self.loadedpkgs[pkgname]=Pkg(pkgname, self)
 311                 markfunction(self.loadedpkgs[pkgname])
 312             else:
 313                 perror('%s not found in source tree' % pkgname)
 314         bailoutonerror()
 315
 316     def __checkbuild(self, marked):
 317         """
 318         Checks queue file if all arches are built
 319
 320         Reads config.builderqueue to grab the info
 321         """
 322         f = urllib.urlopen(config.builderqueue)
 323         requests = {}
 324         reid = re.compile(r'^.*id=(.*) pri.*$')
 325         regb = re.compile(r'^group:.*$|builders:.*$', re.M)
 326         for i in re.findall(regb, f.read()):
 327             if i[0] == 'g':
 328                 id = reid.sub(r'\1', i)
 329                 requests[id] = ""
 330             elif i[0]=='b':
 331                 requests[id] = requests[id] + i
 332         f.close()
 333
 334         for pkg in marked:
 335             for bid in pkg.build.keys():
 336                 if requests.has_key(bid) and not requests[bid].find('?') == -1:
 337                     pkg.error("(buildid %s) building not finished" % bid)
 338
 339     def __checkarchs(self, dsttree, marked):
 340         """
 341         Checks marked pkgs it is built on all archs.
 342         """
 343         for pkg in marked:
 344             if len(pkg.files.keys()) <= 1:
 345                 pkg.error('has only src.rpm built')
 346                 continue
 347             otherpkgnames = self.__find_other_pkgs(pkg, dsttree)
 348
 349             # check if we're not removing some archs
 350             if otherpkgnames:
 351                 curarchs = []
 352                 missingarchs = []
 353                 for somepkg in otherpkgnames:
 354                     curarchs.extend(Pkg(somepkg, dsttree).files.keys())
 355                 for arch in curarchs:
 356                     if arch not in pkg.files.keys():
 357                         missingarchs.append(arch)
 358                 if missingarchs:
 359                     pkg.error('moving would remove archs: %s' % missingarchs)
 360             else:
 361                 # warn if a package isn't built for all archs
 362                 if (config.separate_noarch and 'noarch' in pkg.files.keys() and len(pkg.files.keys()) == 2):
 363                     continue
 364                 elif len(pkg.files.keys()) != len(config.ftp_archs) + 1:
 365                     missingarchs = []
 366                     for arch in config.ftp_archs:
 367                         if arch not in pkg.files.keys():
 368                             missingarchs.append(arch)
 369                     pkg.warning('not built for archs: %s' % missingarchs)
 370
 371     def __rmolderfromsrc(self, test=False):
 372         for pkg in self.marked4moving:
 373             olderpkgnames = self.__find_older_pkgs(pkg)
 374             for i in olderpkgnames:
 375                 Pkg(i, self).remove(test)
 376
 377     def __rmotherfromdst(self, dsttree, test = False):
 378         for pkg in self.marked4moving:
 379             pkgnames = self.__find_other_pkgs(pkg, dsttree)
 380             for i in pkgnames:
 381                 Pkg(i, dsttree).remove(test)
 382
 383     # Used more than once filter functions
 384     def __find_other_pkgs(self, pkg, tree):
 385         escapedpkgname = pkg.name.replace('.', '\.').replace('+', '\+')
 386         ziewre = re.compile(escapedpkgname + '-[^-]*-[^-]*$')
 387         def filter_other_pkgs(x):
 388             if ziewre.match(x) and not x == pkg.nvr:
 389                 return True
 390             else:
 391                 return False
 392         return filter(filter_other_pkgs, tree.pkgnames)
 393
 394     def __find_older_pkgs(self, pkg):
 395         def filter_older_pkgs(x):
 396             c = x.split('-')
 397             rc = rpm.labelCompare(('0', pkg.version, pkg.release),
 398                                                         ('0', c[-2], c[-1]))
 399             if rc == 1: # pkg > x
 400                 return True
 401             else:
 402                 return False
 403         return filter(filter_older_pkgs, self.__find_other_pkgs(pkg, self))
 404