modules/ftptree.py

   1 # vi: encoding=utf-8 ts=8 sts=4 sw=4 et
   2
   3 import os, config, string, urllib, re
   4 from common import fileexists, noarchcachedir
   5 from baseftptree import BasePkg, BaseFtpTree
   6 errnum=0
   7
   8 SomeError="Oh no"
   9 quietmode=False
  10
  11 def bailoutonerror():
  12     if not errnum == 0:
  13         print "%d error(s) encountered... aborting" % errnum
  14         raise SomeError
  15
  16 def pinfo(msg):
  17     print 'INFO: ' + msg
  18
  19 def perror(msg):
  20     global errnum
  21     errnum=errnum+1
  22     print 'ERR: ' + msg
  23
  24 def pwarning(msg):
  25     print 'WARN: ' + msg
  26
  27 def rm(file):
  28     os.remove(file)
  29     #print 'rm: '+file
  30
  31 def mv(src, dst):
  32     os.rename(src, dst+'/'+src.split('/')[-1])
  33     #print "mv: %s %s" % (src, dst+'/'+src.split('/')[-1])
  34
  35 class Pkg(BasePkg):
  36     def __init__(self, nvr, tree):
  37         BasePkg.__init__(self, nvr, tree)
  38         self.name=string.join(nvr.split('-')[:-2], '-')
  39         self.version=nvr.split('-')[-2]
  40         self.release=nvr.split('-')[-1]
  41         self.marked4removal=False
  42         self.marked4moving=False
  43         self.errors=[]
  44         self.warnings=[]
  45
  46     def mark4moving(self):
  47         if not self.marked4moving:
  48             self.tree.marked4moving.append(self)
  49             self.marked4moving=True
  50
  51     def mark4removal(self):
  52         if not self.marked4removal:
  53             self.tree.marked4removal.append(self)
  54             self.marked4removal=True
  55
  56     def error(self, msg):
  57         self.errors.append(msg)
  58         if not quietmode:
  59             perror('%s %s' % (self.nvr, msg))
  60
  61     def warning(self, msg):
  62         self.warnings.append(msg)
  63         if not quietmode:
  64             pwarning('%s %s' % (self.nvr, msg))
  65
  66     def load(self, content=None):
  67         BasePkg.load(self, content)
  68         if self.info.has_key('move'):
  69             self.mark4moving()
  70
  71     def writeinfo(self):
  72         f=open(self.tree.basedir+'/SRPMS/.metadata/'+self.nvr+'.src.rpm.info', 'w')
  73         for bid in self.build.keys():
  74             f.write("info:build:%s:requester:%s\ninfo:build:%s:requester_email:%s\n" % (bid, self.build[bid].requester, bid, self.build[bid].requester_email))
  75         for key in self.info.keys():
  76             f.write("info:%s:%s\n" % (key, string.join(self.info[key], ':')))
  77         for arch in self.files.keys():
  78             for rpm in self.files[arch]:
  79                 f.write("file:%s:%s\n" % (arch, rpm))
  80
  81     def remove(self):
  82         for arch in self.files.keys():
  83             for rpm in self.files[arch]:
  84                 rm(self.tree.basedir+'/'+arch+'/RPMS/'+rpm)
  85                 if arch=='noarch':
  86                     if fileexists(noarchcachedir+rpm+'.filelist'):
  87                         rm(noarchcachedir+rpm+'.filelist')
  88                     if fileexists(noarchcachedir+rpm+'.reqlist'):
  89                         rm(noarchcachedir+rpm+'.reqlist')
  90         rm(self.tree.basedir+'/SRPMS/.metadata/'+self.nvr+'.src.rpm.info')
  91
  92     def move(self, dsttree):
  93         if dsttree.has_key(self.nvr):
  94             movedany=False
  95             for arch in self.files.keys():
  96                 if arch in dsttree[self.nvr].files.keys():
  97                     pinfo("Arch %s for %s is already present in dest tree; removing from srctree" % (arch, self.nvr))
  98                     for rpm in self.files[arch]:
  99                         rm(self.tree.basedir+'/'+arch+'/RPMS/'+rpm)
 100                 else:
 101                     movedany=True
 102                     dsttree[self.nvr].files[arch]=self.files[arch]
 103                     for rpm in self.files[arch]:
 104                         mv(self.tree.basedir+'/'+arch+'/RPMS/'+rpm, dsttree.basedir+'/'+arch+'/RPMS/')
 105             if movedany:
 106                 for bid in self.build.keys():
 107                     dsttree[self.nvr].build[bid]=self.build[bid]
 108                 dsttree[self.nvr].writeinfo()
 109             rm(self.tree.basedir+'/SRPMS/.metadata/'+self.nvr+'.src.rpm.info')
 110         else:
 111             for arch in self.files.keys():
 112                 for rpm in self.files[arch]:
 113                     mv(self.tree.basedir+'/'+arch+'/RPMS/'+rpm, dsttree.basedir+'/'+arch+'/RPMS/')
 114             mv(self.tree.basedir+'/SRPMS/.metadata/'+self.nvr+'.src.rpm.info', dsttree.basedir+'/SRPMS/.metadata/')
 115
 116
 117 class FtpTree(BaseFtpTree):
 118     def __init__(self, tree, loadall=False):
 119         BaseFtpTree.__init__(self, tree)
 120         self.loadedpkgs={}
 121         self.marked4removal=[]
 122         self.marked4moving=[]
 123         self.pkgnames=[]
 124         self.__loadpkgnames()
 125         if loadall:
 126             for pkgname in self.pkgnames:
 127                 self.loadedpkgs[pkgname]=Pkg(pkgname, self)
 128         # Tests:
 129         self.do_checkbuild=True
 130
 131     def __getitem__(self, key):
 132         if self.loadedpkgs.has_key(key):
 133             return self.loadedpkgs[key]
 134         elif key in self.pkgnames:
 135             pkg=Pkg(key, self)
 136             self.loadedpkgs[key]=pkg
 137             return pkg
 138         else:
 139             raise KeyError, key
 140
 141     def has_key(self, key):
 142         if key in self.pkgnames:
 143             return True
 144         else:
 145             return False
 146
 147     def keys(self):
 148         return self.pkgnames
 149
 150     def checktree(self, dsttree):
 151         self.__checkbuild(self.loadedpkgs.values())
 152         self.__checkarchs(dsttree, self.loadedpkgs.values())
 153
 154     def testmove(self, dsttree):
 155         self.__checkbuild(self.marked4moving)
 156         self.__checkarchs(dsttree, self.marked4moving)
 157
 158     def movepkgs(self, dsttree):
 159         if self.do_checkbuild:
 160             self.__checkbuild(self.marked4moving)
 161         bailoutonerror()
 162         self.__checkarchs(dsttree, self.marked4moving)
 163         bailoutonerror()
 164         self.__rmolderfromsrc()
 165         self.__rmotherfromdst(dsttree)
 166
 167         for pkg in self.marked4moving:
 168             pkg.move(dsttree)
 169
 170     def removepkgs(self):
 171         if self.do_checkbuild:
 172             self.__checkbuild(self.marked4removal)
 173         bailoutonerror()
 174         for pkg in self.marked4removal:
 175             pkg.remove()
 176
 177     def mark4removal(self, wannabepkgs):
 178         self.__mark4something(wannabepkgs, Pkg.mark4removal)
 179
 180     def mark4moving(self, wannabepkgs):
 181         self.__mark4something(wannabepkgs, Pkg.mark4moving)
 182
 183
 184     # Internal functions below
 185
 186     def __loadpkgnames(self):
 187         def checkfiletype(name):
 188             if name[-13:]=='.src.rpm.info':
 189                 return True
 190             else:
 191                 return False
 192         list=filter(checkfiletype, os.listdir(self.basedir+'/SRPMS/.metadata'))
 193         self.pkgnames=map((lambda x: x[:-13]), list)
 194
 195     def __mark4something(self, wannabepkgs, markfunction):
 196         def chopoffextension(pkg):
 197             found=pkg.find('.src.rpm')
 198             if found==-1:
 199                 return pkg
 200             else:
 201                 return pkg[:found]
 202         for wannabepkg in wannabepkgs:
 203             pkgname=chopoffextension(wannabepkg)
 204             if pkgname in self.pkgnames:
 205                 if not pkgname in self.loadedpkgs.keys():
 206                     self.loadedpkgs[pkgname]=Pkg(pkgname, self)
 207                 markfunction(self.loadedpkgs[pkgname])
 208             else:
 209                 perror('%s not found in source tree' % pkgname)
 210         bailoutonerror()
 211
 212     def __checkbuild(self, marked):
 213         f=urllib.urlopen(config.builderqueue)
 214         #f=open('queue.txt')
 215         requests={}
 216         reid=re.compile(r'^.*id=(.*) pri.*$')
 217         regb=re.compile(r'^group:.*$|builders:.*$', re.M)
 218         for i in re.findall(regb, f.read()):
 219             if i[0]=='g':
 220                 id=reid.sub(r'\1', i)
 221                 requests[id]=""
 222             elif i[0]=='b':
 223                 requests[id]=requests[id]+i
 224         f.close()
 225         for pkg in marked:
 226             for bid in pkg.build.keys():
 227                 if requests.has_key(bid) and not requests[bid].find('?') == -1:
 228                     pkg.error("(buildid %s) building not finished" % bid)
 229
 230     def __checkarchs(self, dsttree, marked):
 231         for pkg in marked:
 232             if len(pkg.files.keys()) <= 1:
 233                 pkg.error('has only src.rpm built')
 234                 continue
 235             otherpkgnames=self.__find_other_pkgs(pkg, dsttree)
 236             if otherpkgnames: # check if we're not removing some archs
 237                 curarchs=[]
 238                 missingarchs=[]
 239                 for somepkg in otherpkgnames:
 240                     curarchs.extend(Pkg(somepkg, dsttree).files.keys())
 241                 for arch in curarchs:
 242                     if arch not in pkg.files.keys():
 243                         missingarchs.append(arch)
 244                 if missingarchs:
 245                     pkg.error('moving would remove archs: %s' % missingarchs)
 246             else: # warn if a package isn't built for all archs
 247                 if (config.separate_noarch and 'noarch' in pkg.files.keys() and
 248                                     len(pkg.files.keys())==2):
 249                     continue
 250                 elif len(pkg.files.keys()) != len(config.ftp_archs)+1:
 251                     missingarchs=[]
 252                     for arch in config.ftp_archs:
 253                         if arch not in pkg.files.keys():
 254                             missingarchs.append(arch)
 255                     pkg.warning('not built for archs: %s' % missingarchs)
 256
 257     def __rmolderfromsrc(self):
 258         for pkg in self.marked4moving:
 259             olderpkgnames=self.__find_older_pkgs(pkg)
 260             for i in olderpkgnames:
 261                 Pkg(i, self).remove()
 262
 263     def __rmotherfromdst(self, dsttree):
 264         for pkg in self.marked4moving:
 265             pkgnames=self.__find_other_pkgs(pkg, dsttree)
 266             for i in pkgnames:
 267                 Pkg(i, dsttree).remove()
 268
 269     # Used more than once filter functions
 270
 271     def __find_other_pkgs(self, pkg, tree):
 272         escapedpkgname=pkg.name.replace('.', '\.').replace('+', '\+')
 273         ziewre=re.compile(escapedpkgname+'-[^-]*-[^-]*$')
 274         def filter_other_pkgs(x):
 275             if ziewre.match(x) and not x == pkg.nvr:
 276                 return True
 277             else:
 278                 return False
 279         return filter(filter_other_pkgs, tree.pkgnames)
 280
 281     def __find_older_pkgs(self, pkg):
 282         def filter_older_pkgs(x):
 283             checking=x.split('-')
 284             if checking[-2]<pkg.version:
 285                 return True
 286             elif checking[-2]==pkg.version:
 287                 if checking[-1]<pkg.release:
 288                     return True
 289                 else:
 290                     return False
 291             else:
 292                 return False
 293         return filter(filter_older_pkgs, self.__find_other_pkgs(pkg, self))
 294
 295