modules/ftptree.py

   1 # vi: encoding=utf-8 ts=8 sts=4 sw=4 et
   2
   3 import os, config, string, urllib, re, rpm
   4 from common import fileexists, noarchcachedir
   5 from baseftptree import BasePkg, BaseFtpTree
   6 errnum=0
   7
   8 SomeError="Oh no"
   9 quietmode=False
  10
  11 def bailoutonerror():
  12     if not errnum == 0:
  13         print "%d error(s) encountered... aborting" % errnum
  14         raise SomeError
  15
  16 def pinfo(msg):
  17     print 'INFO: ' + msg
  18
  19 def perror(msg):
  20     global errnum
  21     errnum=errnum+1
  22     print 'ERR: ' + msg
  23
  24 def pwarning(msg):
  25     print 'WARN: ' + msg
  26
  27 def rm(file, test=False):
  28     if test:
  29         if not os.path.exists(file):
  30             pinfo("TEST os.remove(%s): file doesn't exists" % file)
  31     else:
  32         os.remove(file)
  33         #print 'rm: '+file
  34
  35 def mv(src, dst, test=False):
  36     fsrc = src
  37     fdst = dst+'/'+src.split('/')[-1]
  38     if test:
  39         if not os.path.exists(src):
  40             pinfo("TEST os.rename(%s, %s): source doesn't exists" % (fsrc, fdst))
  41     else:
  42         try:
  43             os.rename(fsrc, fdst)
  44         except OSError, e:
  45             pinfo("os.rename(%s, %s): %s" % (fsrc, fdst, e))
  46             raise
  47
  48 class Pkg(BasePkg):
  49     def __init__(self, nvr, tree):
  50         BasePkg.__init__(self, nvr, tree)
  51         self.name=string.join(nvr.split('-')[:-2], '-')
  52         self.version=nvr.split('-')[-2]
  53         self.release=nvr.split('-')[-1]
  54         self.marked4removal=False
  55         self.marked4moving=False
  56         self.marked4movingpool=[]
  57         self.errors=[]
  58         self.warnings=[]
  59
  60     def __cmp__(self, pkg):
  61         if self.name > pkg.name:
  62             return 1
  63         elif self.name < pkg.name:
  64             return -1
  65         else:
  66             return rpm.labelCompare(('0', self.version, self.release),
  67                                     ('0', pkg.version, pkg.release))
  68
  69     def mark4moving(self):
  70         if not self.marked4moving:
  71             # Only one pkg in this pool can be marked for moving
  72             for pkg in self.marked4movingpool:
  73                 pkg.unmark4moving()
  74             self.tree.marked4moving.append(self)
  75             self.marked4moving=True
  76
  77     def unmark4moving(self):
  78         if self.marked4moving:
  79             self.tree.marked4moving.remove(self)
  80             self.marked4moving=False
  81
  82     def mark4removal(self):
  83         if not self.marked4removal:
  84             self.tree.marked4removal.append(self)
  85             self.marked4removal=True
  86
  87     def error(self, msg):
  88         self.errors.append(msg)
  89         if not quietmode:
  90             perror('%s %s' % (self.nvr, msg))
  91
  92     def warning(self, msg):
  93         self.warnings.append(msg)
  94         if not quietmode:
  95             pwarning('%s %s' % (self.nvr, msg))
  96
  97     def load(self, content=None):
  98         BasePkg.load(self, content)
  99         if self.info.has_key('move'):
 100             self.mark4moving()
 101
 102     def writeinfo(self):
 103         f=open(self.tree.basedir+'/SRPMS/.metadata/'+self.nvr+'.src.rpm.info', 'w')
 104         for bid in self.build.keys():
 105             f.write("info:build:%s:requester:%s\ninfo:build:%s:requester_email:%s\n" % (bid, self.build[bid].requester, bid, self.build[bid].requester_email))
 106         for key in self.info.keys():
 107             f.write("info:%s:%s\n" % (key, string.join(self.info[key], ':')))
 108         for arch in self.files.keys():
 109             for rpm in self.files[arch]:
 110                 f.write("file:%s:%s\n" % (arch, rpm))
 111
 112     def remove(self, test=False):
 113         for arch in self.files.keys():
 114             for rpm in self.files[arch]:
 115                 rm(self.tree.basedir+'/'+arch+'/RPMS/'+rpm, test)
 116                 if arch=='noarch':
 117                     if fileexists(noarchcachedir+rpm+'.filelist'):
 118                         rm(noarchcachedir+rpm+'.filelist', test)
 119                     if fileexists(noarchcachedir+rpm+'.reqlist'):
 120                         rm(noarchcachedir+rpm+'.reqlist', test)
 121         rm(self.tree.basedir+'/SRPMS/.metadata/'+self.nvr+'.src.rpm.info', test)
 122
 123     def move(self, dsttree, test=False):
 124         if dsttree.has_key(self.nvr):
 125             movedany=False
 126             for arch in self.files.keys():
 127                 if arch in dsttree[self.nvr].files.keys():
 128                     msg = ""
 129                     if test:
 130                         msg = "TEST "
 131                     pinfo("%sArch %s for %s is already present in dest tree; removing from srctree" % (msg, arch, self.nvr))
 132                     for rpm in self.files[arch]:
 133                         rm(self.tree.basedir+'/'+arch+'/RPMS/'+rpm, test)
 134                 else:
 135                     movedany=True
 136                     dsttree[self.nvr].files[arch]=self.files[arch]
 137                     for rpm in self.files[arch]:
 138                         mv(self.tree.basedir+'/'+arch+'/RPMS/'+rpm, dsttree.basedir+'/'+arch+'/RPMS/', test)
 139             if not test and  movedany:
 140                 for bid in self.build.keys():
 141                     dsttree[self.nvr].build[bid]=self.build[bid]
 142                 dsttree[self.nvr].writeinfo()
 143             rm(self.tree.basedir+'/SRPMS/.metadata/'+self.nvr+'.src.rpm.info', test)
 144         else:
 145             for arch in self.files.keys():
 146                 for rpm in self.files[arch]:
 147                     mv(self.tree.basedir+'/'+arch+'/RPMS/'+rpm, dsttree.basedir+'/'+arch+'/RPMS/', test)
 148             mv(self.tree.basedir+'/SRPMS/.metadata/'+self.nvr+'.src.rpm.info', dsttree.basedir+'/SRPMS/.metadata/', test)
 149
 150
 151 class FtpTree(BaseFtpTree):
 152     def __init__(self, tree, loadall=False):
 153         BaseFtpTree.__init__(self, tree)
 154         self.loadedpkgs={}
 155         self.marked4removal=[]
 156         self.marked4moving=[]
 157         self.pkgnames=[]
 158         self.__loadpkgnames()
 159         if loadall:
 160             for pkgname in self.pkgnames:
 161                 self.loadedpkgs[pkgname]=Pkg(pkgname, self)
 162         # Tests:
 163         self.do_checkbuild=True
 164
 165     def __getitem__(self, key):
 166         if self.loadedpkgs.has_key(key):
 167             return self.loadedpkgs[key]
 168         elif key in self.pkgnames:
 169             pkg=Pkg(key, self)
 170             self.loadedpkgs[key]=pkg
 171             return pkg
 172         else:
 173             raise KeyError, key
 174
 175     def has_key(self, key):
 176         if key in self.pkgnames:
 177             return True
 178         else:
 179             return False
 180
 181     def keys(self):
 182         return self.pkgnames
 183
 184     def values(self):
 185         return self.loadedpkgs.values()
 186
 187     def checktree(self, dsttree):
 188         self.__checkbuild(self.loadedpkgs.values())
 189         self.__checkarchs(dsttree, self.loadedpkgs.values())
 190
 191     def testmove(self, dsttree):
 192         self.__checkbuild(self.marked4moving)
 193         self.__checkarchs(dsttree, self.marked4moving)
 194
 195         self.__rmolderfromsrc(test=True)
 196         self.__rmotherfromdst(dsttree, test=True)
 197
 198         for pkg in self.marked4moving:
 199             pkg.move(dsttree, test=True)
 200
 201     def movepkgs(self, dsttree):
 202         if self.do_checkbuild:
 203             self.__checkbuild(self.marked4moving)
 204         bailoutonerror()
 205         self.__checkarchs(dsttree, self.marked4moving)
 206         bailoutonerror()
 207         self.__rmolderfromsrc()
 208         self.__rmotherfromdst(dsttree)
 209
 210         for pkg in self.marked4moving:
 211             pkg.move(dsttree)
 212
 213     def removepkgs(self):
 214         if self.do_checkbuild:
 215             self.__checkbuild(self.marked4removal)
 216         bailoutonerror()
 217         for pkg in self.marked4removal:
 218             pkg.remove()
 219
 220     def mark4removal(self, wannabepkgs):
 221         self.__mark4something(wannabepkgs, Pkg.mark4removal)
 222
 223     def mark4moving(self, wannabepkgs):
 224         self.__mark4something(wannabepkgs, Pkg.mark4moving)
 225
 226
 227     # Internal functions below
 228
 229     def __loadpkgnames(self):
 230         def checkfiletype(name):
 231             if name[-13:]=='.src.rpm.info':
 232                 return True
 233             else:
 234                 return False
 235         list=filter(checkfiletype, os.listdir(self.basedir+'/SRPMS/.metadata'))
 236         self.pkgnames=map((lambda x: x[:-13]), list)
 237
 238     def __mark4something(self, wannabepkgs, markfunction):
 239         def chopoffextension(pkg):
 240             found=pkg.find('.src.rpm')
 241             if found==-1:
 242                 return pkg
 243             else:
 244                 return pkg[:found]
 245         for wannabepkg in wannabepkgs:
 246             pkgname=chopoffextension(wannabepkg)
 247             if pkgname in self.pkgnames:
 248                 if not pkgname in self.loadedpkgs.keys():
 249                     self.loadedpkgs[pkgname]=Pkg(pkgname, self)
 250                 markfunction(self.loadedpkgs[pkgname])
 251             else:
 252                 perror('%s not found in source tree' % pkgname)
 253         bailoutonerror()
 254
 255     def __checkbuild(self, marked):
 256         f=urllib.urlopen(config.builderqueue)
 257         #f=open('queue.txt')
 258         requests={}
 259         reid=re.compile(r'^.*id=(.*) pri.*$')
 260         regb=re.compile(r'^group:.*$|builders:.*$', re.M)
 261         for i in re.findall(regb, f.read()):
 262             if i[0]=='g':
 263                 id=reid.sub(r'\1', i)
 264                 requests[id]=""
 265             elif i[0]=='b':
 266                 requests[id]=requests[id]+i
 267         f.close()
 268         for pkg in marked:
 269             for bid in pkg.build.keys():
 270                 if requests.has_key(bid) and not requests[bid].find('?') == -1:
 271                     pkg.error("(buildid %s) building not finished" % bid)
 272
 273     def __checkarchs(self, dsttree, marked):
 274         for pkg in marked:
 275             if len(pkg.files.keys()) <= 1:
 276                 pkg.error('has only src.rpm built')
 277                 continue
 278             otherpkgnames=self.__find_other_pkgs(pkg, dsttree)
 279             if otherpkgnames: # check if we're not removing some archs
 280                 curarchs=[]
 281                 missingarchs=[]
 282                 for somepkg in otherpkgnames:
 283                     curarchs.extend(Pkg(somepkg, dsttree).files.keys())
 284                 for arch in curarchs:
 285                     if arch not in pkg.files.keys():
 286                         missingarchs.append(arch)
 287                 if missingarchs:
 288                     pkg.error('moving would remove archs: %s' % missingarchs)
 289             else: # warn if a package isn't built for all archs
 290                 if (config.separate_noarch and 'noarch' in pkg.files.keys() and
 291                                     len(pkg.files.keys())==2):
 292                     continue
 293                 elif len(pkg.files.keys()) != len(config.ftp_archs)+1:
 294                     missingarchs=[]
 295                     for arch in config.ftp_archs:
 296                         if arch not in pkg.files.keys():
 297                             missingarchs.append(arch)
 298                     pkg.warning('not built for archs: %s' % missingarchs)
 299
 300     def __rmolderfromsrc(self, test=False):
 301         for pkg in self.marked4moving:
 302             olderpkgnames=self.__find_older_pkgs(pkg)
 303             for i in olderpkgnames:
 304                 Pkg(i, self).remove(test)
 305
 306     def __rmotherfromdst(self, dsttree, test=False):
 307         for pkg in self.marked4moving:
 308             pkgnames=self.__find_other_pkgs(pkg, dsttree)
 309             for i in pkgnames:
 310                 Pkg(i, dsttree).remove(test)
 311
 312     # Used more than once filter functions
 313
 314     def __find_other_pkgs(self, pkg, tree):
 315         escapedpkgname=pkg.name.replace('.', '\.').replace('+', '\+')
 316         ziewre=re.compile(escapedpkgname+'-[^-]*-[^-]*$')
 317         def filter_other_pkgs(x):
 318             if ziewre.match(x) and not x == pkg.nvr:
 319                 return True
 320             else:
 321                 return False
 322         return filter(filter_other_pkgs, tree.pkgnames)
 323
 324     def __find_older_pkgs(self, pkg):
 325         def filter_older_pkgs(x):
 326             c=x.split('-')
 327             rc = rpm.labelCompare(('0', pkg.version, pkg.release),
 328                                                         ('0', c[-2], c[-1]))
 329             if rc == 1: # pkg > x
 330                 return True
 331             else:
 332                 return False
 333         return filter(filter_older_pkgs, self.__find_other_pkgs(pkg, self))
 334