modules/ftptree.py

   1 # vi: encoding=utf-8 ts=8 sts=4 sw=4 et
   2
   3 import os, config, string, urllib, re, rpm
   4 from common import fileexists, noarchcachedir
   5 from baseftptree import BasePkg, BaseFtpTree
   6 errnum=0
   7
   8 quietmode=False
   9
  10 class SomeError(Exception):
  11     def __init__(self):
  12         return
  13
  14     def __str__(self):
  15         print "","An Error occured!"
  16
  17 def bailoutonerror():
  18     if not errnum == 0:
  19         print "%d error(s) encountered... aborting" % errnum
  20         raise SomeError
  21
  22 def pinfo(msg):
  23     print 'INFO: ' + msg
  24
  25 def perror(msg):
  26     global errnum
  27     errnum=errnum+1
  28     print 'ERR: ' + msg
  29
  30 def pwarning(msg):
  31     print 'WARN: ' + msg
  32
  33 def rm(file, test=False):
  34     if test:
  35         if not os.path.exists(file):
  36             pinfo("TEST os.remove(%s): file doesn't exists" % file)
  37     else:
  38         try:
  39             os.remove(file)
  40         except OSError, e:
  41             pinfo("os.remove(%s): %s" % (file, e))
  42             #raise
  43
  44 def mv(src, dst, test=False):
  45     fsrc = src
  46     fdst = dst+'/'+src.split('/')[-1]
  47     if test:
  48         if not os.path.exists(src):
  49             pinfo("TEST os.rename(%s, %s): source doesn't exists" % (fsrc, fdst))
  50     else:
  51         try:
  52             os.rename(fsrc, fdst)
  53         except OSError, e:
  54             pinfo("os.rename(%s, %s): %s" % (fsrc, fdst, e))
  55             raise
  56
  57 class Pkg(BasePkg):
  58     def __init__(self, nvr, tree):
  59         BasePkg.__init__(self, nvr, tree)
  60         self.name = string.join(nvr.split('-')[:-2], '-')
  61         self.version = nvr.split('-')[-2]
  62         self.release = nvr.split('-')[-1]
  63         self.marked4removal = False
  64         self.marked4moving = False
  65         self.marked4movingpool = []
  66         self.errors = []
  67         self.warnings = []
  68
  69     def __cmp__(self, pkg):
  70         if self.name > pkg.name:
  71             return 1
  72         elif self.name < pkg.name:
  73             return -1
  74         else:
  75             return rpm.labelCompare(('0', self.version, self.release),
  76                                     ('0', pkg.version, pkg.release))
  77
  78     def mark4moving(self):
  79         if not self.marked4moving:
  80             # Only one pkg in this pool can be marked for moving
  81             for pkg in self.marked4movingpool:
  82                 pkg.unmark4moving()
  83             self.tree.marked4moving.append(self)
  84             self.marked4moving=True
  85
  86     def unmark4moving(self):
  87         if self.marked4moving:
  88             self.tree.marked4moving.remove(self)
  89             self.marked4moving=False
  90
  91     def mark4removal(self):
  92         if not self.marked4removal:
  93             self.tree.marked4removal.append(self)
  94             self.marked4removal=True
  95
  96     def error(self, msg):
  97         self.errors.append(msg)
  98         if not quietmode:
  99             perror('%s %s' % (self.nvr, msg))
 100
 101     def warning(self, msg):
 102         self.warnings.append(msg)
 103         if not quietmode:
 104             pwarning('%s %s' % (self.nvr, msg))
 105
 106     def load(self, content=None):
 107         BasePkg.load(self, content)
 108         if self.info.has_key('move'):
 109             self.mark4moving()
 110
 111     def writeinfo(self):
 112         f = open(self.tree.basedir+'/SRPMS/.metadata/'+self.nvr+'.src.rpm.info', 'w')
 113         for bid in self.build.keys():
 114             f.write("info:build:%s:requester:%s\ninfo:build:%s:requester_email:%s\n" % (bid, self.build[bid].requester, bid, self.build[bid].requester_email))
 115         for key in self.info.keys():
 116             f.write("info:%s:%s\n" % (key, string.join(self.info[key], ':')))
 117         for arch in self.files.keys():
 118             for rpm in self.files[arch]:
 119                 f.write("file:%s:%s\n" % (arch, rpm))
 120
 121     def remove(self, test = False):
 122         """
 123         Remove package from ftp
 124         """
 125         for arch in self.files.keys():
 126             for rpm in self.files[arch]:
 127                 rm(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, test)
 128                 if arch == 'noarch':
 129                     if fileexists(noarchcachedir + rpm + '.filelist'):
 130                         rm(noarchcachedir + rpm + '.filelist', test)
 131                     if fileexists(noarchcachedir + rpm + '.reqlist'):
 132                         rm(noarchcachedir + rpm + '.reqlist', test)
 133         rm(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', test)
 134
 135     def move(self, dsttree, test=False):
 136         if dsttree.has_key(self.nvr):
 137             movedany = False
 138             for arch in self.files.keys():
 139                 if arch in dsttree[self.nvr].files.keys():
 140                     msg = ""
 141                     if test:
 142                         msg = "TEST "
 143                     pinfo("%sArch %s for %s is already present in dest tree; removing from srctree" % (msg, arch, self.nvr))
 144                     for rpm in self.files[arch]:
 145                         rm(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, test)
 146                 else:
 147                     movedany = True
 148                     dsttree[self.nvr].files[arch] = self.files[arch]
 149                     for rpm in self.files[arch]:
 150                         mv(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, dsttree.basedir + '/' + arch + '/RPMS/', test)
 151             if not test and  movedany:
 152                 for bid in self.build.keys():
 153                     dsttree[self.nvr].build[bid] = self.build[bid]
 154                 dsttree[self.nvr].writeinfo()
 155             rm(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', test)
 156         else:
 157             # move files
 158             for arch in self.files.keys():
 159                 for rpm in self.files[arch]:
 160                     mv(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, dsttree.basedir + '/' + arch + '/RPMS/', test)
 161
 162             # move metadata
 163             mv(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', dsttree.basedir + '/SRPMS/.metadata/', test)
 164
 165 class FtpTree(BaseFtpTree):
 166     def __init__(self, tree, loadall=False):
 167         BaseFtpTree.__init__(self, tree)
 168         self.loadedpkgs = {}
 169         self.marked4removal = []
 170         self.marked4moving = []
 171         self.pkgnames = []
 172         self.__loadpkgnames()
 173         if loadall:
 174             for pkgname in self.pkgnames:
 175                 self.loadedpkgs[pkgname] = Pkg(pkgname, self)
 176         # Tests:
 177         self.do_checkbuild = True
 178
 179     def __getitem__(self, key):
 180         if self.loadedpkgs.has_key(key):
 181             return self.loadedpkgs[key]
 182         elif key in self.pkgnames:
 183             pkg=Pkg(key, self)
 184             self.loadedpkgs[key]=pkg
 185             return pkg
 186         else:
 187             raise KeyError, key
 188
 189     def has_key(self, key):
 190         if key in self.pkgnames:
 191             return True
 192         else:
 193             return False
 194
 195     def keys(self):
 196         return self.pkgnames
 197
 198     def values(self):
 199         return self.loadedpkgs.values()
 200
 201     def checktree(self, dsttree):
 202         self.__checkbuild(self.loadedpkgs.values())
 203         self.__checkarchs(dsttree, self.loadedpkgs.values())
 204
 205     def testmove(self, dsttree):
 206         self.__checkbuild(self.marked4moving)
 207         self.__checkarchs(dsttree, self.marked4moving)
 208
 209         self.__rmolderfromsrc(test = True)
 210         self.__rmotherfromdst(dsttree, test = True)
 211
 212         for pkg in self.marked4moving:
 213             pkg.move(dsttree, test = True)
 214
 215     def movepkgs(self, dsttree):
 216         if self.do_checkbuild:
 217             self.__checkbuild(self.marked4moving)
 218         bailoutonerror()
 219         self.__checkarchs(dsttree, self.marked4moving)
 220         bailoutonerror()
 221         self.__rmolderfromsrc()
 222         self.__rmotherfromdst(dsttree)
 223
 224         for pkg in self.marked4moving:
 225             pkg.move(dsttree)
 226
 227     def removepkgs(self):
 228         if self.do_checkbuild:
 229             self.__checkbuild(self.marked4removal)
 230         bailoutonerror()
 231         for pkg in self.marked4removal:
 232             pkg.remove()
 233
 234     def mark4removal(self, wannabepkgs):
 235         self.__mark4something(wannabepkgs, Pkg.mark4removal)
 236
 237     def mark4moving(self, wannabepkgs):
 238         self.__mark4something(wannabepkgs, Pkg.mark4moving)
 239
 240
 241     # Internal functions below
 242
 243     def __loadpkgnames(self):
 244         def checkfiletype(name):
 245             if name[-13:]=='.src.rpm.info':
 246                 return True
 247             else:
 248                 return False
 249         list = filter(checkfiletype, os.listdir(self.basedir+'/SRPMS/.metadata'))
 250         self.pkgnames = map((lambda x: x[:-13]), list)
 251
 252     def __mark4something(self, wannabepkgs, markfunction):
 253         def chopoffextension(pkg):
 254             found=pkg.find('.src.rpm')
 255             if found==-1:
 256                 return pkg
 257             else:
 258                 return pkg[:found]
 259         for wannabepkg in wannabepkgs:
 260             pkgname=chopoffextension(wannabepkg)
 261             if pkgname in self.pkgnames:
 262                 if not pkgname in self.loadedpkgs.keys():
 263                     self.loadedpkgs[pkgname]=Pkg(pkgname, self)
 264                 markfunction(self.loadedpkgs[pkgname])
 265             else:
 266                 perror('%s not found in source tree' % pkgname)
 267         bailoutonerror()
 268
 269     def __checkbuild(self, marked):
 270         """
 271         Checks queue file if all arches are built
 272
 273         Reads config.builderqueue to grab the info
 274         """
 275         f = urllib.urlopen(config.builderqueue)
 276         requests = {}
 277         reid = re.compile(r'^.*id=(.*) pri.*$')
 278         regb = re.compile(r'^group:.*$|builders:.*$', re.M)
 279         for i in re.findall(regb, f.read()):
 280             if i[0] == 'g':
 281                 id = reid.sub(r'\1', i)
 282                 requests[id] = ""
 283             elif i[0]=='b':
 284                 requests[id] = requests[id] + i
 285         f.close()
 286
 287         for pkg in marked:
 288             for bid in pkg.build.keys():
 289                 if requests.has_key(bid) and not requests[bid].find('?') == -1:
 290                     pkg.error("(buildid %s) building not finished" % bid)
 291
 292     def __checkarchs(self, dsttree, marked):
 293         """
 294         Checks marked pkgs it is built on all archs.
 295         """
 296         for pkg in marked:
 297             if len(pkg.files.keys()) <= 1:
 298                 pkg.error('has only src.rpm built')
 299                 continue
 300             otherpkgnames = self.__find_other_pkgs(pkg, dsttree)
 301
 302             # check if we're not removing some archs
 303             if otherpkgnames:
 304                 curarchs = []
 305                 missingarchs = []
 306                 for somepkg in otherpkgnames:
 307                     curarchs.extend(Pkg(somepkg, dsttree).files.keys())
 308                 for arch in curarchs:
 309                     if arch not in pkg.files.keys():
 310                         missingarchs.append(arch)
 311                 if missingarchs:
 312                     pkg.error('moving would remove archs: %s' % missingarchs)
 313             else:
 314                 # warn if a package isn't built for all archs
 315                 if (config.separate_noarch and 'noarch' in pkg.files.keys() and len(pkg.files.keys()) == 2):
 316                     continue
 317                 elif len(pkg.files.keys()) != len(config.ftp_archs) + 1:
 318                     missingarchs = []
 319                     for arch in config.ftp_archs:
 320                         if arch not in pkg.files.keys():
 321                             missingarchs.append(arch)
 322                     pkg.warning('not built for archs: %s' % missingarchs)
 323
 324     def __rmolderfromsrc(self, test=False):
 325         for pkg in self.marked4moving:
 326             olderpkgnames = self.__find_older_pkgs(pkg)
 327             for i in olderpkgnames:
 328                 Pkg(i, self).remove(test)
 329
 330     def __rmotherfromdst(self, dsttree, test = False):
 331         for pkg in self.marked4moving:
 332             pkgnames = self.__find_other_pkgs(pkg, dsttree)
 333             for i in pkgnames:
 334                 Pkg(i, dsttree).remove(test)
 335
 336     # Used more than once filter functions
 337     def __find_other_pkgs(self, pkg, tree):
 338         escapedpkgname = pkg.name.replace('.', '\.').replace('+', '\+')
 339         ziewre = re.compile(escapedpkgname + '-[^-]*-[^-]*$')
 340         def filter_other_pkgs(x):
 341             if ziewre.match(x) and not x == pkg.nvr:
 342                 return True
 343             else:
 344                 return False
 345         return filter(filter_other_pkgs, tree.pkgnames)
 346
 347     def __find_older_pkgs(self, pkg):
 348         def filter_older_pkgs(x):
 349             c = x.split('-')
 350             rc = rpm.labelCompare(('0', pkg.version, pkg.release),
 351                                                         ('0', c[-2], c[-1]))
 352             if rc == 1: # pkg > x
 353                 return True
 354             else:
 355                 return False
 356         return filter(filter_older_pkgs, self.__find_other_pkgs(pkg, self))
 357