#!/usr/bin/env python
# vi: encoding=utf-8 ts=8 sts=4 sw=4 et
-import sys, os, stat, time
+from __future__ import print_function
+
+import sys, os, stat, time, re
sys.path.insert(0, os.environ['HOME']+'/pld-ftp-admin/modules')
from config import incoming_dir, default_to, ftp_archs
+from config import value as cval
import config
from common import noarchcachedir, tmpdir, fileexists
from baseftptree import BaseFtpTree, BasePkg
-cval=config.value
+from ftptree import FtpTree, Pkg
import ftpio
-try:
- ftpio.connect('from-incoming-pid-%s' % os.getpid())
-except:
- sys.exit(0)
+os.umask(0o022)
def rm(file):
os.remove(file)
- #print 'rm: '+file
def mv(src, dst):
- os.rename(src, dst+'/'+src.split('/')[-1])
- #print "mv: %s %s" % (src, dst+'/'+src.split('/')[-1])
+ os.rename(src, dst + '/' + src.split('/')[-1])
+
+# duplicate code in ftptree.py
+def is_debuginfo(nvr):
+ """
+ returns true if NVR is debuginfo package and separate debuginfo is enabled
+ """
+ if not config.separate_debuginfo:
+ return False
+ pkg = nvr.split('-')[:-2]
+ return pkg[-1] == 'debuginfo'
def findfiles(dir):
def filterinfos(x):
- if x[-11:]=='.uploadinfo':
- mtime = os.stat(dir + '/' + x)[ST_MTIME]
- now = time.time()
- # don't look into uploadinfo files for 30 seconds after scp
- if now - mtime > 30:
- return True
- else:
- return False
+ if x[-11:] == '.uploadinfo':
+ return True
else:
return False
return filter(filterinfos, os.listdir(dir))
def getcontent(file):
- f=open(file, 'r')
- content=f.read()
+ f = open(file, 'r')
+ content = f.read()
f.close()
- if not content[-5:]=='\nEND\n':
+ if not content[-5:] == '\nEND\n':
return None
else:
return content[:-4]
-ftptree=BaseFtpTree(cval['default_to'])
-
-if not ftpio.lock(cval['default_to']):
- sys.exit(0)
-
-for uploadinfo in findfiles(incoming_dir+'SRPMS'):
- content=getcontent(incoming_dir+'SRPMS/'+uploadinfo)
- if not content:
- continue # Uploading not finished
-
- pkg=BasePkg(uploadinfo[:-19], content=content)
- srpm=pkg.files['SRPMS'][0]
-
- if ftptree.has_key(`pkg`):
- ftpio.log("%s already present in %s; removing newer files" %
- (srpm, ftptree))
- rm(incoming_dir+'SRPMS/'+srpm)
- f=open(default_to+'SRPMS/.metadata/'+srpm+'.info', 'a')
- bid=pkg.build.keys()[0]
- build=pkg.build[bid]
- f.write("info:build:%s:requester:%s\ninfo:build:%s:requester_email:%s\n"
- % (bid, build.requester, bid, build.requester_email))
- f.close()
- else:
- mv(incoming_dir+'SRPMS/'+srpm, default_to+'SRPMS/RPMS')
-
- f=open(default_to+'SRPMS/.metadata/'+srpm+'.info', 'w')
- f.write(content)
- f.close()
-
- rm(incoming_dir+'SRPMS/'+uploadinfo)
-
-def send_noarch_msg(files_differ, reqs_differ, pkg, rpm, arch):
+def send_noarch_msg(files_differ, reqs_differ, pkg, rpmfile, arch):
req_email=pkg.build[pkg.lastbid].requester_email
req_bid=pkg.lastbid
cc_list=[]
if 'logs_list' in cval:
cc_list.append(cval['logs_list'])
- m_subject="Noarch error: %s files differ among builders" % rpm
+ m_subject="Noarch error: %s files differ among builders" % rpmfile
bids=pkg.build.keys()
if len(bids)>1:
for bid in bids:
""" % (cval['xpldbuilder'], req_bid, req_bid, m_subject)
sm = os.popen("/usr/sbin/sendmail -t", "w")
-
+
sm.write(msg)
if files_differ:
f=open("%s/files.diff" % tmpdir, 'r')
sm.write("Difference between %s (currently in %s) and %s FILES\n" %
- (pkg.noarch_arch[rpm], `ftptree`, arch)),
+ (pkg.noarch_arch[rpmfile], `ftptree`, arch)),
for line in f.readlines(True)[2:]:
sm.write(line)
f.close()
if reqs_differ:
f=open("%s/reqs.diff" % tmpdir, 'r')
sm.write("Difference between %s (currently in %s) and %s REQS\n" %
- (pkg.noarch_arch[rpm], `ftptree`, arch)),
+ (pkg.noarch_arch[rpmfile], `ftptree`, arch)),
for line in f.readlines(True)[2:]:
sm.write(line)
f.close()
sm.close()
-
-
-def move_noarch(f, arch, rpm, dstpkg):
- if dstpkg.noarch_arch.has_key(rpm):
- os.system("rpm -qlp %s > %s/files.new" %
- (incoming_dir+arch+'/'+rpm, tmpdir))
- os.system("rpm -qRp %s |sort|uniq > %s/reqs.new" %
- (incoming_dir+arch+'/'+rpm, tmpdir))
+def move_noarch(f, arch, rpmfile, dstpkg):
+ if dstpkg.noarch_arch.has_key(rpmfile):
+ os.system("LC_ALL=C rpm -qlp %s | LC_ALL=C sort > %s/files.new" %
+ (incoming_dir + arch + '/' + rpmfile, tmpdir))
+ os.system("rpm -qRp %s | LC_ALL=C sort | LC_ALL=C uniq > %s/reqs.new" %
+ (incoming_dir + arch + '/' + rpmfile, tmpdir))
- files_differ=False
- reqs_differ=False
+ files_differ = False
+ reqs_differ = False
if os.system("diff -u %s/%s.filelist %s/files.new > %s/files.diff" %
- (noarchcachedir, rpm, tmpdir, tmpdir)):
- files_differ=True
+ (noarchcachedir, rpmfile, tmpdir, tmpdir)):
+ files_differ = True
if os.system("diff -u %s/%s.reqlist %s/reqs.new > %s/reqs.diff" %
- (noarchcachedir, rpm, tmpdir, tmpdir)):
- reqs_differ=True
+ (noarchcachedir, rpmfile, tmpdir, tmpdir)):
+ reqs_differ = True
if files_differ or reqs_differ:
- send_noarch_msg(files_differ, reqs_differ, dstpkg, rpm, arch)
+ send_noarch_msg(files_differ, reqs_differ, dstpkg, rpmfile, arch)
+
+ rm(incoming_dir + arch + '/' + rpmfile)
+ else:
+ os.system("LC_ALL=C rpm -qlp %s | LC_ALL=C sort > %s/%s.filelist" %
+ (incoming_dir + arch + '/' + rpmfile, noarchcachedir, rpmfile))
+ os.system("rpm -qRp %s | LC_ALL=C sort | LC_ALL=C uniq > %s/%s.reqlist" %
+ (incoming_dir + arch + '/' + rpmfile, noarchcachedir, rpmfile))
+ if not dstpkg.files.has_key(arch):
+ f.write("file:noarch:%s\ninfo:noarch_arch:%s:%s\n" % (rpmfile, rpmfile, arch))
+ mv(incoming_dir + arch + '/' + rpmfile, default_to + 'noarch/RPMS')
+
+def send_vr_msg(snvr, anvr, pkg, arch):
+ req_email=pkg.build[pkg.lastbid].requester_email
+ req_bid=pkg.lastbid
+ cc_list=[]
+ if 'logs_list' in cval:
+ cc_list.append(cval['logs_list'])
+ m_subject="NVR error: %s version or relese differ among subpackages" % snvr[0]
+ bids=pkg.build.keys()
+ if len(bids)>1:
+ for bid in bids:
+ newcc=pkg.build[bid].requester_email
+ if req_email!=newcc and newcc not in cc_list:
+ cc_list.append(newcc)
+
+ msg="From: %s\nTo: %s\n" % (cval['from_field'], req_email)
+ if cc_list:
+ msg=msg+"Cc: %s\n" % ", ".join(cc_list)
+ msg=msg+"""X-PLD-Builder: %s
+References: <%s@pld.src.builder>
+In-Reply-To: <%s@pld.src.builder>
+Subject: %s
+
+""" % (cval['xpldbuilder'], req_bid, req_bid, m_subject)
+
+ sm = os.popen("/usr/sbin/sendmail -t", "w")
+
+ sm.write(msg)
+
+ sm.write("Difference between %s SRPM (currently in %s) and %s RPM NVR:\n\n" %
+ (snvr[0], `ftptree`, arch)),
+ sm.write("Expected (%s):\nV: %s\nR: %s\n\n" % snvr)
+ sm.write("RPM:\nN: %s\nV: %s\nR: %s\n" % anvr)
+ sm.write('\n')
+
+ sm.close()
+
+# main()
+try:
+ ftpio.connect('from-incoming-pid-%s' % os.getpid())
+except:
+ print("Can't get ftpiod connection")
+ sys.exit(1)
+
+ftptree = BaseFtpTree(cval['default_to'])
+
+if not ftpio.lock(cval['default_to']):
+ print("Can't get lock: %s" % cval['default_to'])
+ sys.exit(1)
+
+moved_anything = False
+
+for uploadinfo in findfiles(incoming_dir + 'SRPMS'):
+ content = getcontent(incoming_dir + 'SRPMS/' + uploadinfo)
+ if not content:
+ continue # Uploading not finished
+
+ pkg = BasePkg(uploadinfo[:-19], content = content)
+ srpm = pkg.files['SRPMS'][0]
- rm(incoming_dir+arch+'/'+rpm)
+ if not os.path.exists(incoming_dir + 'SRPMS/' + srpm):
+ ftpio.log("%s file missing; skipping move until next round" % (srpm))
+ continue
+
+ if ftptree.has_key(`pkg`):
+ ftpio.log("%s already present in %s; removing older files" % (srpm, ftptree))
+ rm(default_to + 'SRPMS/RPMS/' + srpm)
+ f = open(default_to + 'SRPMS/.metadata/' + srpm+'.info', 'a')
+ bid = pkg.build.keys()[0]
+ build = pkg.build[bid]
+ f.write("info:build:%s:requester:%s\ninfo:build:%s:requester_email:%s\n"
+ % (bid, build.requester, bid, build.requester_email))
+ f.close()
else:
- os.system("rpm -qlp %s > %s/%s.filelist" %
- (incoming_dir+arch+'/'+rpm, noarchcachedir, rpm))
- os.system("rpm -qRp %s |sort|uniq > %s/%s.reqlist" %
- (incoming_dir+arch+'/'+rpm, noarchcachedir, rpm))
- f.write("file:noarch:%s\ninfo:noarch_arch:%s:%s\n" % (rpm, rpm, arch))
- mv(incoming_dir+arch+'/'+rpm, default_to+'noarch/RPMS')
+ f = open(default_to + 'SRPMS/.metadata/' + srpm + '.info', 'w')
+ f.write(content)
+ f.close()
+
+ mv(incoming_dir + 'SRPMS/' + srpm, default_to + 'SRPMS/RPMS')
+ rm(incoming_dir + 'SRPMS/' + uploadinfo)
for arch in ftp_archs:
- for uploadinfo in findfiles(incoming_dir+arch):
- content=getcontent(incoming_dir+arch+'/'+uploadinfo)
+ for uploadinfo in findfiles(incoming_dir + arch):
+ content = getcontent(incoming_dir + arch + '/' + uploadinfo)
if not content:
+ ftpio.log("%s not finished uploading" % uploadinfo)
continue # Uploading not finished
- srcpkg=BasePkg(uploadinfo[:-19], content=content)
- srpm=srcpkg.files['SRPMS'][0]
+ srcpkg = BasePkg(uploadinfo[:-19], content = content)
+ srpm = srcpkg.files['SRPMS'][0]
if not ftptree.has_key(`srcpkg`):
continue # We require the src.rpm to be present
- dstpkg=BasePkg(`srcpkg`, ftptree)
+ renvr = re.compile(r'(.*)-(.*)-(.*)\.[^.]*\.rpm')
+ srcnvr = renvr.match(srpm).groups()
+
+ rpmfile_missing = [f for f in srcpkg.files['ARCH'] if not os.path.exists(incoming_dir + arch + '/'+f)]
+ if len(rpmfile_missing):
+ for filem in rpmfile_missing:
+ ftpio.log("%s file missing; skipping move until next round" % (filem))
+ continue
+
+ dstpkg = BasePkg(`srcpkg`, ftptree)
if dstpkg.files.has_key(arch):
- ftpio.log("files from %s for arch %s already present in %s; removing newer files" % (`srcpkg`, arch, ftptree))
- for rpm in srcpkg.files['ARCH']:
+ ftpio.log("files from %s for arch %s already present in %s; removing older files" % (`srcpkg`, arch, ftptree))
+ for rpmfile in dstpkg.files[arch]:
+ if is_debuginfo(rpmfile):
+ dstfile = default_to + arch + '/debuginfo'
+ else:
+ dstfile = default_to + arch + '/RPMS'
try:
- rm(incoming_dir+arch+'/'+rpm)
- except OSError, e:
- l = "Removing %s problem: %s" % (incoming_dir+arch+'/'+rpm, e)
+ rm(dstfile + '/' + rpmfile)
+ except OSError as e:
+ l = "Removing %s problem: %s" % (dstfile + '/' + rpmfile, e)
ftpio.log(l)
- print l
- rm(incoming_dir+arch+'/'+uploadinfo)
- continue
+ print(l)
+
+ f = open(default_to + 'SRPMS/.metadata/' + srpm + '.info', 'a')
+ for rpmfile in srcpkg.files['ARCH']:
+ moved_anything = True
- f=open(default_to+'SRPMS/.metadata/'+srpm+'.info', 'a')
- for rpm in srcpkg.files['ARCH']:
- if rpm[-11:]=='.noarch.rpm' and config.separate_noarch:
- move_noarch(f, arch, rpm, dstpkg)
+# Too much noise, too little use
+# archnvr = renvr.match(rpmfile).groups()
+# if srcnvr[1] != archnvr[1] or srcnvr[2] != archnvr[2]:
+# send_vr_msg(srcnvr, archnvr, dstpkg, arch)
+
+ if rpmfile[-11:] == '.noarch.rpm' and config.separate_noarch:
+ move_noarch(f, arch, rpmfile, dstpkg)
else:
- f.write("file:%s:%s\n" % (arch, rpm))
- mv(incoming_dir+arch+'/'+rpm, default_to+arch+'/RPMS')
+ if not dstpkg.files.has_key(arch):
+ f.write("file:%s:%s\n" % (arch, rpmfile))
+ srcfile = incoming_dir + arch + '/' + rpmfile
+
+ if is_debuginfo(rpmfile):
+ dstfile = default_to + arch + '/debuginfo'
+ else:
+ dstfile = default_to + arch + '/RPMS'
+
+ try:
+ mv(srcfile, dstfile)
+ except OSError as e:
+ l = "Moving %s to %s problem: %s" % (srcfile, dstfile, e)
+ ftpio.log(l)
+ print(l)
f.close()
- rm(incoming_dir+arch+'/'+uploadinfo)
+ rm(incoming_dir + arch + '/' + uploadinfo)
ftpio.unlock(cval['default_to'])
+if moved_anything:
+ os.system("%s/pld-ftp-admin/bin/pfa-genindex --quiet test > /dev/null" % (os.getenv("HOME")))
projects / projects / pld-ftp-admin.git / blobdiff