1 # vi: encoding=utf-8 ts=8 sts=4 sw=4 et
3 import os, config, string, urllib, re, rpm
4 from common import fileexists, noarchcachedir
5 from baseftptree import BasePkg, BaseFtpTree
6 from sign import is_signed
11 class SomeError(Exception):
16 print "","An Error occured!"
20 print "%d error(s) encountered... aborting" % errnum
34 def rm(file, test = False):
36 if not os.path.exists(file):
37 pinfo("TEST os.remove(%s): file doesn't exists" % file)
42 pinfo("os.remove(%s): %s" % (file, e))
45 def mv(src, dst, test = False):
47 fdst = dst + '/' + src.split('/')[-1]
49 if not os.path.exists(fsrc):
50 pinfo("TEST os.rename(%s, %s): source doesn't exists" % (fsrc, fdst))
51 if not os.path.exists(dst):
52 pinfo("TEST destination doesn't exist: %s" % dst)
57 pinfo("os.rename(%s, %s): %s" % (fsrc, fdst, e))
61 def __init__(self, nvr, tree):
62 BasePkg.__init__(self, nvr, tree)
63 self.name = string.join(nvr.split('-')[:-2], '-')
64 self.version = nvr.split('-')[-2]
65 self.release = nvr.split('-')[-1]
66 self.marked4removal = False
67 self.marked4moving = False
68 self.marked4movingpool = []
72 def __cmp__(self, pkg):
73 if self.name > pkg.name:
75 elif self.name < pkg.name:
78 return rpm.labelCompare(('0', self.version, self.release),
79 ('0', pkg.version, pkg.release))
82 # unfortunately can't do new Pkg(NVR), and have no "tree" in this pkg context
83 # so this static function
84 def is_debuginfo(self, nvr):
86 returns true if NVR is debuginfo package and separate debuginfo is enabled
88 if not config.separate_debuginfo:
90 pkg = nvr.split('-')[:-2]
91 return pkg[-1] == 'debuginfo'
93 def is_sourcefile(self, file):
95 returns true if file is source package
97 return file[-8:] == '.src.rpm'
99 # returns true if package build is integer
100 def is_release(self):
102 To account Release tags with subver macros, we consider integer release
103 if it contains odd number of dots:
107 0.%{subver}.%{rel}, %{rel} = 1 -> 0.20010.1 -> True
108 0.%{subver}.%{rel}, %{rel} = 0.1 -> 0.20010.0.1 -> False
110 return self.release.count('.') % 2 == 0
112 def mark4moving(self):
113 if not self.marked4moving:
114 # Only one pkg in this pool can be marked for moving
115 for pkg in self.marked4movingpool:
117 self.tree.marked4moving.append(self)
118 self.marked4moving=True
120 def unmark4moving(self):
121 if self.marked4moving:
122 self.tree.marked4moving.remove(self)
123 self.marked4moving=False
125 def mark4removal(self):
126 if not self.marked4removal:
127 self.tree.marked4removal.append(self)
128 self.marked4removal=True
130 def error(self, msg):
131 self.errors.append(msg)
133 perror('%s %s' % (self.nvr, msg))
135 def warning(self, msg):
136 self.warnings.append(msg)
138 pwarning('%s %s' % (self.nvr, msg))
140 def load(self, content=None):
141 BasePkg.load(self, content)
142 if self.info.has_key('move'):
146 f = open(self.tree.basedir+'/SRPMS/.metadata/'+self.nvr+'.src.rpm.info', 'w')
147 for bid in self.build.keys():
148 f.write("info:build:%s:requester:%s\ninfo:build:%s:requester_email:%s\n" % (bid, self.build[bid].requester, bid, self.build[bid].requester_email))
149 for key in self.info.keys():
150 f.write("info:%s:%s\n" % (key, string.join(self.info[key], ':')))
151 for arch in self.files.keys():
152 for rpm in self.files[arch]:
153 f.write("file:%s:%s\n" % (arch, rpm))
155 def remove(self, test = False):
157 Remove package from ftp
159 for arch in self.files.keys():
160 for rpm in self.files[arch]:
161 if self.is_debuginfo(rpm):
162 rm(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, test)
164 rm(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, test)
166 if fileexists(noarchcachedir + rpm + '.filelist'):
167 rm(noarchcachedir + rpm + '.filelist', test)
168 if fileexists(noarchcachedir + rpm + '.reqlist'):
169 rm(noarchcachedir + rpm + '.reqlist', test)
170 rm(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', test)
172 def rpmfiles(self, debugfiles = True, sourcefiles = True):
174 Return rpm files related to this package
177 for arch, rpms in self.files.items():
179 if self.is_debuginfo(nvr):
181 files.append(self.tree.basedir + '/' + arch + '/debuginfo/' + nvr)
183 if self.is_sourcefile(nvr):
185 files.append(self.tree.basedir + '/' + arch + '/RPMS/' + nvr)
187 files.append(self.tree.basedir + '/' + arch + '/RPMS/' + nvr)
192 Return obsoletes for all packages in Pkg:
194 {'php-geshi': set(['geshi'])}
199 ts.setVSFlags(rpm.RPMVSF_NODSAHEADER)
200 fdno = os.open(pkg, os.O_RDONLY)
201 hdr = ts.hdrFromFdno(fdno)
206 for rpmfile in self.rpmfiles():
207 if not os.path.exists(rpmfile):
209 hdr = rpmhdr(rpmfile)
210 if not hdr[rpm.RPMTAG_OBSOLETES]:
213 name = hdr[rpm.RPMTAG_NAME]
214 if not name in obsoletes:
215 obsoletes[name] = set()
217 for tag in hdr[rpm.RPMTAG_OBSOLETES]:
218 obsoletes[name].add(tag)
222 def move(self, dsttree, test = False):
223 if dsttree.has_key(self.nvr):
225 for arch in self.files.keys():
226 if arch in dsttree[self.nvr].files.keys():
230 pinfo("%sArch %s for %s is already present in dest tree; removing from srctree" % (msg, arch, self.nvr))
231 for rpm in self.files[arch]:
232 if self.is_debuginfo(rpm):
233 rm(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, test)
235 rm(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, test)
238 dsttree[self.nvr].files[arch] = self.files[arch]
239 for rpm in self.files[arch]:
240 if self.is_debuginfo(rpm):
241 mv(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, dsttree.basedir + '/' + arch + '/debuginfo/', test)
243 mv(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, dsttree.basedir + '/' + arch + '/RPMS/', test)
244 if not test and movedany:
245 for bid in self.build.keys():
246 dsttree[self.nvr].build[bid] = self.build[bid]
247 dsttree[self.nvr].writeinfo()
248 rm(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', test)
251 for arch in self.files.keys():
252 for rpm in self.files[arch]:
253 if self.is_debuginfo(rpm):
254 mv(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, dsttree.basedir + '/' + arch + '/debuginfo/', test)
256 mv(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, dsttree.basedir + '/' + arch + '/RPMS/', test)
259 mv(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', dsttree.basedir + '/SRPMS/.metadata/', test)
261 class FtpTree(BaseFtpTree):
262 def __init__(self, tree, loadall=False):
263 BaseFtpTree.__init__(self, tree)
265 self.marked4removal = []
266 self.marked4moving = []
268 self.__loadpkgnames()
270 for pkgname in self.pkgnames:
271 self.loadedpkgs[pkgname] = Pkg(pkgname, self)
273 self.do_checkbuild = True
275 def __getitem__(self, key):
276 if self.loadedpkgs.has_key(key):
277 return self.loadedpkgs[key]
278 elif key in self.pkgnames:
280 self.loadedpkgs[key]=pkg
285 def has_key(self, key):
286 if key in self.pkgnames:
295 return self.loadedpkgs.values()
297 def checktree(self, dsttree):
298 self.__checkbuild(self.loadedpkgs.values())
299 self.__checkarchs(dsttree, self.loadedpkgs.values())
301 def testmove(self, dsttree, archivetree = None):
302 self.__checkbuild(self.marked4moving)
303 self.__checkarchs(dsttree, self.marked4moving)
304 if not dsttree.treename.count("archive"):
305 self.__checkduplicates(self.marked4moving)
307 self.__checksigns(dsttree, self.marked4moving, test = True)
308 self.__checkforobsoletes(dsttree, self.marked4moving, test = True)
309 self.__checkforrelease(dsttree, self.marked4moving, test = True)
311 if not self.treename.count("archive"):
312 self.__rmolderfromsrc(test = True)
313 if not dsttree.treename.count("archive"):
314 self.__rmotherfromdst(dsttree, test = True, archivetree = archivetree)
316 for pkg in self.marked4moving:
317 pkg.move(dsttree, test = True)
319 def movepkgs(self, dsttree, archivetree = None):
320 if self.do_checkbuild:
321 self.__checkbuild(self.marked4moving)
324 self.__checkarchs(dsttree, self.marked4moving)
327 self.__checksigns(dsttree, self.marked4moving)
330 if not self.treename.count("archive"):
331 self.__rmolderfromsrc()
332 if not dsttree.treename.count("archive"):
333 self.__rmotherfromdst(dsttree, archivetree = archivetree)
335 for pkg in self.marked4moving:
338 def rpmfiles(self, debugfiles = True, sourcefiles = True):
339 if self.do_checkbuild:
340 self.__checkbuild(self.marked4moving)
343 for pkg in self.marked4moving:
344 files += pkg.rpmfiles(debugfiles = debugfiles, sourcefiles = sourcefiles)
347 def removepkgs(self):
348 if self.do_checkbuild:
349 self.__checkbuild(self.marked4removal)
351 for pkg in self.marked4removal:
354 def mark4removal(self, wannabepkgs):
355 self.__mark4something(wannabepkgs, Pkg.mark4removal)
357 def mark4moving(self, wannabepkgs):
358 self.__mark4something(wannabepkgs, Pkg.mark4moving)
360 # Internal functions below
361 def __arch_stringify(self, list):
363 dist = config.ftp_dist;
365 ret.append(dist + '-' + arch)
368 def __loadpkgnames(self):
369 def checkfiletype(name):
370 if name[-13:]=='.src.rpm.info':
374 list = filter(checkfiletype, os.listdir(self.basedir+'/SRPMS/.metadata'))
375 self.pkgnames = map((lambda x: x[:-13]), list)
377 def __mark4something(self, wannabepkgs, markfunction):
378 def chopoffextension(pkg):
379 found = pkg.find('.src.rpm')
385 for wannabepkg in wannabepkgs:
386 pkgname = chopoffextension(wannabepkg)
387 if pkgname in self.pkgnames:
388 if not pkgname in self.loadedpkgs.keys():
389 self.loadedpkgs[pkgname]=Pkg(pkgname, self)
390 markfunction(self.loadedpkgs[pkgname])
392 perror('%s not found in source tree' % pkgname)
395 def __checkbuild(self, marked):
397 Checks queue file if all arches are built
399 Reads config.builderqueue to grab the info
401 f = urllib.urlopen(config.builderqueue)
403 reid = re.compile(r'^.*id=(.*) pri.*$')
404 regb = re.compile(r'^group:.*$|builders:.*$', re.M)
405 for i in re.findall(regb, f.read()):
407 id = reid.sub(r'\1', i)
410 requests[id] = requests[id] + i
414 for bid in pkg.build.keys():
415 if requests.has_key(bid) and not requests[bid].find('?') == -1:
416 pkg.error("(buildid %s) building not finished" % bid)
418 def __checkarchs(self, dsttree, marked):
420 Checks marked pkgs it is built on all archs.
423 if len(pkg.files.keys()) <= 1:
424 pkg.error('has only src.rpm built')
426 otherpkgnames = self.__find_other_pkgs(pkg, dsttree)
428 # check if we're not removing some archs
432 for somepkg in otherpkgnames:
433 curarchs.extend(Pkg(somepkg, dsttree).files.keys())
434 for arch in curarchs:
435 if arch not in pkg.files.keys():
436 missingarchs.append(arch)
438 pkg.error('moving would remove archs: %s' % self.__arch_stringify(missingarchs))
440 # warn if a package isn't built for all archs
442 ftp_archs_num = len(config.ftp_archs) + 1
443 if (config.separate_noarch and 'noarch' in pkg.files.keys()):
444 # ftp_archs + SRPMS + noarch subpackages
446 # plain simple noarch package
447 if (len(pkg.files.keys()) == 2):
450 if len(pkg.files.keys()) != ftp_archs_num:
452 for arch in config.ftp_archs:
453 if arch not in pkg.files.keys():
454 missingarchs.append(arch)
455 pkg.warning('not built for archs: %s' % self.__arch_stringify(missingarchs))
457 def __checkduplicates(self, marked):
459 Checks if marked packages contain duplicate packages (with different versions)
462 olderpkgnames = self.__find_older_pkgs(pkg)
463 for i in olderpkgnames:
464 markednames = [str(x) for x in marked]
466 pkg.error('duplicate package: %s' % i)
468 def __rmolderfromsrc(self, test = False):
469 for pkg in self.marked4moving:
470 olderpkgnames = self.__find_older_pkgs(pkg)
471 for i in olderpkgnames:
472 Pkg(i, self).remove(test)
474 def __rmotherfromdst(self, dsttree, test = False, archivetree = None):
475 for pkg in self.marked4moving:
476 pkgnames = self.__find_other_pkgs(pkg, dsttree)
478 if archivetree == None:
479 Pkg(i, dsttree).remove(test)
481 Pkg(i, dsttree).move(archivetree, test = test)
483 # Used more than once filter functions
484 def __find_other_pkgs(self, pkg, tree):
485 escapedpkgname = pkg.name.replace('.', '\.').replace('+', '\+')
486 ziewre = re.compile(escapedpkgname + '-[^-]*-[^-]*$')
487 def filter_other_pkgs(x):
488 if ziewre.match(x) and not x == pkg.nvr:
492 return filter(filter_other_pkgs, tree.pkgnames)
494 def __find_older_pkgs(self, pkg):
495 def filter_older_pkgs(x):
497 rc = rpm.labelCompare(('0', pkg.version, pkg.release),
499 if rc == 1: # pkg > x
503 return filter(filter_older_pkgs, self.__find_other_pkgs(pkg, self))
505 def __checksigns(self, tree, pkgs, test = False):
507 Checks if pkgs in tree are all signed.
509 in case of test = true, error flag is set for unsigned packages
511 if not tree.treename in config.signed_trees:
516 for file in pkg.rpmfiles():
517 if not is_signed(file):
523 pkg.warning('%d files not signed' % unsigned)
525 pkg.error('%d files not signed' % unsigned)
527 def __checkforobsoletes(self, tree, pkgs, test = False):
529 Checks queue file if package obsoletes something in destination tree and suggest for removal.
531 Only NAME tag is compared, i.e virtual packages do not get reported.
537 def findbyname(name):
539 return '-'.join(nvr.split('-')[:-2]) == name
540 return filter(x, tree.pkgnames)
543 obsoletes = pkg.obsoletes()
547 for pn, setlist in obsoletes.items():
551 pkg.warning('obsoletes %s (via %s) in dest tree, perhaps you want rmpkg' % (p,pn))
553 def __checkforrelease(self, tree, pkgs, test = False):
555 Checks queue file if package release is non integer.
562 if not pkg.is_release():
563 pkg.warning('non-integer release: %s' % pkg.release)