modules/ftptree.py

   1 # vi: encoding=utf-8 ts=8 sts=4 sw=4 et
   2
   3 import os, config, string, urllib, re
   4 from common import fileexists, noarchcachedir
   5 from baseftptree import BasePkg, BaseFtpTree
   6 errnum=0
   7
   8 SomeError="Oh no"
   9
  10 def bailoutonerror():
  11     if not errnum == 0:
  12         print "%d error(s) encountered... aborting" % errnum
  13         raise SomeError
  14
  15 def pinfo(msg):
  16     print 'INFO: ' + msg
  17
  18 def perror(msg):
  19     global errnum
  20     errnum=errnum+1
  21     print 'ERR: ' + msg
  22
  23 def pwarning(msg):
  24     print 'WARN: ' + msg
  25
  26 def rm(file):
  27     os.remove(file)
  28     #print 'rm: '+file
  29
  30 def mv(src, dst):
  31     os.rename(src, dst+'/'+src.split('/')[-1])
  32     #print "mv: %s %s" % (src, dst+'/'+src.split('/')[-1])
  33
  34 class Pkg(BasePkg):
  35     def __init__(self, name, tree):
  36         self.marked4removal=False
  37         self.marked4moving=False
  38         self.errors=[]
  39         self.warnings=[]
  40         BasePkg.__init__(self, name, tree)
  41
  42     def mark4moving(self):
  43         if not self.marked4moving:
  44             self.tree.marked4moving.append(self)
  45             self.marked4moving=True
  46
  47     def mark4removal(self):
  48         if not self.marked4removal:
  49             self.tree.marked4removal.append(self)
  50             self.marked4removal=True
  51
  52     def load(self, content=None):
  53         BasePkg.load(self, content)
  54         if self.info.has_key('move'):
  55             self.mark4moving()
  56
  57     def writeinfo(self):
  58         f=open(self.tree.basedir+'/SRPMS/.metadata/'+self.name+'.src.rpm.info', 'w')
  59         for bid in self.build.keys():
  60             f.write("info:build:%s:requester:%s\ninfo:build:%s:requester_email:%s\n" % (bid, self.build[bid].requester, bid, self.build[bid].requester_email))
  61         for key in self.info.keys():
  62             f.write("info:%s:%s\n" % (key, string.join(self.info[key], ':')))
  63         for arch in self.files.keys():
  64             for rpm in self.files[arch]:
  65                 f.write("file:%s:%s\n" % (arch, rpm))
  66
  67     def remove(self):
  68         for arch in self.files.keys():
  69             for rpm in self.files[arch]:
  70                 rm(self.tree.basedir+'/'+arch+'/RPMS/'+rpm)
  71                 if arch=='noarch':
  72                     if fileexists(noarchcachedir+rpm+'.filelist'):
  73                         rm(noarchcachedir+rpm+'.filelist')
  74                     if fileexists(noarchcachedir+rpm+'.reqlist'):
  75                         rm(noarchcachedir+rpm+'.reqlist')
  76         rm(self.tree.basedir+'/SRPMS/.metadata/'+self.name+'.src.rpm.info')
  77
  78     def move(self, dsttree):
  79         if dsttree.has_key(self.name):
  80             movedany=False
  81             for arch in self.files.keys():
  82                 if arch in dsttree[self.name].files.keys():
  83                     pinfo("Arch %s for %s is already present in dest tree; removing from srctree" % (arch, self.name))
  84                     for rpm in self.files[arch]:
  85                         rm(self.tree.basedir+'/'+arch+'/RPMS/'+rpm)
  86                 else:
  87                     movedany=True
  88                     dsttree[self.name].files[arch]=self.files[arch]
  89                     for rpm in self.files[arch]:
  90                         mv(self.tree.basedir+'/'+arch+'/RPMS/'+rpm, dsttree.basedir+'/'+arch+'/RPMS/')
  91             if movedany:
  92                 for bid in self.build.keys():
  93                     dsttree[self.name].build[bid]=self.build[bid]
  94                 dsttree[self.name].writeinfo()
  95             rm(self.tree.basedir+'/SRPMS/.metadata/'+self.name+'.src.rpm.info')
  96         else:
  97             for arch in self.files.keys():
  98                 for rpm in self.files[arch]:
  99                     mv(self.tree.basedir+'/'+arch+'/RPMS/'+rpm, dsttree.basedir+'/'+arch+'/RPMS/')
 100             mv(self.tree.basedir+'/SRPMS/.metadata/'+self.name+'.src.rpm.info', dsttree.basedir+'/SRPMS/.metadata/')
 101
 102
 103 class FtpTree(BaseFtpTree):
 104     def __init__(self, tree, loadall=False):
 105         BaseFtpTree.__init__(self, tree)
 106         self.loadedpkgs={}
 107         self.marked4removal=[]
 108         self.marked4moving=[]
 109         self.pkgnames=[]
 110         self.__loadpkgnames()
 111         if loadall:
 112             for pkgname in self.pkgnames:
 113                 self.loadedpkgs[pkgname]=Pkg(pkgname, self)
 114         # Tests:
 115         self.do_checkbuild=True
 116
 117     def __getitem__(self, key):
 118         if self.loadedpkgs.has_key(key):
 119             return self.loadedpkgs[key]
 120         elif key in self.pkgnames:
 121             pkg=Pkg(key, self)
 122             self.loadedpkgs[key]=pkg
 123             return pkg
 124         else:
 125             raise KeyError, key
 126
 127     def has_key(self, key):
 128         if key in self.pkgnames:
 129             return True
 130         else:
 131             return False
 132
 133     def keys(self):
 134         return self.pkgnames
 135
 136     def checktree(self, dsttree):
 137         pass
 138
 139     def testmove(self, dsttree):
 140         self.__checkbuild(self.marked4moving)
 141         self.__checkarchs(dsttree, self.marked4moving)
 142
 143     def movepkgs(self, dsttree):
 144         if self.do_checkbuild:
 145             self.__checkbuild(self.marked4moving)
 146         bailoutonerror()
 147         self.__checkarchs(dsttree, self.marked4moving)
 148         bailoutonerror()
 149         self.__rmolderfromsrc()
 150         self.__rmotherfromdst(dsttree)
 151
 152         for pkg in self.marked4moving:
 153             pkg.move(dsttree)
 154
 155     def removepkgs(self):
 156         if self.do_checkbuild:
 157             self.__checkbuild(self.marked4removal)
 158         bailoutonerror()
 159         for pkg in self.marked4removal:
 160             pkg.remove()
 161
 162     def mark4removal(self, wannabepkgs):
 163         self.__mark4something(wannabepkgs, Pkg.mark4removal)
 164
 165     def mark4moving(self, wannabepkgs):
 166         self.__mark4something(wannabepkgs, Pkg.mark4moving)
 167
 168
 169     # Internal functions below
 170
 171     def __loadpkgnames(self):
 172         def checkfiletype(name):
 173             if name[-13:]=='.src.rpm.info':
 174                 return True
 175             else:
 176                 return False
 177         list=filter(checkfiletype, os.listdir(self.basedir+'/SRPMS/.metadata'))
 178         self.pkgnames=map((lambda x: x[:-13]), list)
 179
 180     def __mark4something(self, wannabepkgs, markfunction):
 181         def chopoffextension(pkg):
 182             found=pkg.find('.src.rpm')
 183             if found==-1:
 184                 return pkg
 185             else:
 186                 return pkg[:found]
 187         for wannabepkg in wannabepkgs:
 188             pkgname=chopoffextension(wannabepkg)
 189             if pkgname in self.pkgnames:
 190                 if not pkgname in self.loadedpkgs.keys():
 191                     self.loadedpkgs[pkgname]=Pkg(pkgname, self)
 192                 markfunction(self.loadedpkgs[pkgname])
 193             else:
 194                 perror('%s not found in source tree' % pkgname)
 195         bailoutonerror()
 196
 197     def __checkbuild(self, marked):
 198         f=urllib.urlopen(config.builderqueue)
 199         #f=open('queue.txt')
 200         requests={}
 201         reid=re.compile(r'^.*id=(.*) pri.*$')
 202         regb=re.compile(r'^group:.*$|builders:.*$', re.M)
 203         for i in re.findall(regb, f.read()):
 204             if i[0]=='g':
 205                 id=reid.sub(r'\1', i)
 206                 requests[id]=""
 207             elif i[0]=='b':
 208                 requests[id]=requests[id]+i
 209         f.close()
 210         for pkg in marked:
 211             for bid in pkg.build.keys():
 212                 if requests.has_key(bid) and not requests[bid].find('?') == -1:
 213                     perror("%s (buildid %s) building not finished" % (pkg,bid))
 214
 215     def __checkarchs(self, dsttree, marked):
 216         for pkg in marked:
 217             if len(pkg.files.keys()) <= 1:
 218                 perror('%s has only src.rpm built' % pkg)
 219                 continue
 220             otherpkgnames=self.__find_other_pkgs(pkg, dsttree)
 221             if otherpkgnames: # check if we're not removing some archs
 222                 curarchs=[]
 223                 missingarchs=[]
 224                 for somepkg in otherpkgnames:
 225                     curarchs.extend(Pkg(somepkg, dsttree).files.keys())
 226                 for arch in curarchs:
 227                     if arch not in pkg.files.keys():
 228                         missingarchs.append(arch)
 229                 if missingarchs:
 230                     perror('%s moving would remove archs: %s' %
 231                                                             (pkg, missingarchs))
 232             else: # warn if a package isn't built for all archs
 233                 if (config.separate_noarch and 'noarch' in pkg.files.keys() and
 234                                     len(pkg.files.keys())==2):
 235                     continue
 236                 elif len(pkg.files.keys()) != len(config.ftp_archs)+1:
 237                     missingarchs=[]
 238                     for arch in config.ftp_archs:
 239                         if arch not in pkg.files.keys():
 240                             missingarchs.append(arch)
 241                     pwarning('%s not built for archs: %s' %
 242                                                             (pkg, missingarchs))
 243
 244     def __rmolderfromsrc(self):
 245         for pkg in self.marked4moving:
 246             olderpkgnames=self.__find_older_pkgs(pkg)
 247             for i in olderpkgnames:
 248                 Pkg(i, self).remove()
 249
 250     def __rmotherfromdst(self, dsttree):
 251         for pkg in self.marked4moving:
 252             pkgnames=self.__find_other_pkgs(pkg, dsttree)
 253             for i in pkgnames:
 254                 Pkg(i, dsttree).remove()
 255
 256     # Used more than once filter functions
 257
 258     def __find_other_pkgs(self, pkg, tree):
 259         pkgname=string.join(pkg.name.split('-')[:-2], '-')
 260         escapedpkgname=pkgname.replace('.', '\.').replace('+', '\+')
 261         ziewre=re.compile(escapedpkgname+'-[^-]*-[^-]*$')
 262         def filter_other_pkgs(x):
 263             if ziewre.match(x) and not x == pkg.name:
 264                 return True
 265             else:
 266                 return False
 267         return filter(filter_other_pkgs, tree.pkgnames)
 268
 269     def __find_older_pkgs(self, pkg):
 270         def filter_older_pkgs(x):
 271             checking=x.split('-')
 272             curpkg=pkg.name.split('-')
 273             if checking[-2]<curpkg[-2]:
 274                 return True
 275             elif checking[-2]==curpkg[-2]:
 276                 if checking[-1]<curpkg[-1]:
 277                     return True
 278                 else:
 279                     return False
 280             else:
 281                 return False
 282         return filter(filter_older_pkgs, self.__find_other_pkgs(pkg, self))
 283
 284