[projects/pld-ftp-admin.git] / bin / pfa-from-incoming

#!/usr/bin/env python
# vi: encoding=utf-8 ts=8 sts=4 sw=4 et

import sys, os, stat, time
sys.path.insert(0, os.environ['HOME']+'/pld-ftp-admin/modules')
from config import incoming_dir, default_to, ftp_archs
from config import value as cval
import config
from common import noarchcachedir, tmpdir, fileexists
from baseftptree import BaseFtpTree, BasePkg
from ftptree import FtpTree, Pkg
import ftpio

def rm(file):
    os.remove(file)

def mv(src, dst):
    os.rename(src, dst + '/' + src.split('/')[-1])

# duplicate code in ftptree.py
def is_debuginfo(nvr):
    """
    returns true if NVR is debuginfo package and separate debuginfo is enabled
    """
    if not config.separate_debuginfo:
        return False
    pkg = nvr.split('-')[:-2]
    return pkg[-1] == 'debuginfo'

def findfiles(dir):
    def filterinfos(x):
        if x[-11:] == '.uploadinfo':
            return True
        else:
            return False
    return filter(filterinfos, os.listdir(dir))

def getcontent(file):
    f = open(file, 'r')
    content = f.read()
    f.close()
    if not content[-5:] == '\nEND\n':
        return None
    else:
        return content[:-4]

def send_noarch_msg(files_differ, reqs_differ, pkg, rpmfile, arch):
    req_email=pkg.build[pkg.lastbid].requester_email
    req_bid=pkg.lastbid
    cc_list=[]
    if 'logs_list' in cval:
        cc_list.append(cval['logs_list'])
    m_subject="Noarch error: %s files differ among builders" % rpmfile
    bids=pkg.build.keys()
    if len(bids)>1:
        for bid in bids:
            newcc=pkg.build[bid].requester_email
            if req_email!=newcc and newcc not in cc_list:
                cc_list.append(newcc)

    msg="From: %s\nTo: %s\n" % (cval['from_field'], req_email)
    if cc_list:
        msg=msg+"Cc: %s\n" % ", ".join(cc_list)
    msg=msg+"""X-PLD-Builder: %s
References: <%s@pld.src.builder>
In-Reply-To: <%s@pld.src.builder>
Subject: %s

""" % (cval['xpldbuilder'], req_bid, req_bid, m_subject)

    sm = os.popen("/usr/sbin/sendmail -t", "w")

    sm.write(msg)

    if files_differ:
        f=open("%s/files.diff" % tmpdir, 'r')
        sm.write("Difference between %s (currently in %s) and %s FILES\n" %
              (pkg.noarch_arch[rpmfile], `ftptree`, arch)),
        for line in f.readlines(True)[2:]:
            sm.write(line)
        f.close()

    sm.write('\n')

    if reqs_differ:
        f=open("%s/reqs.diff" % tmpdir, 'r')
        sm.write("Difference between %s (currently in %s) and %s REQS\n" %
              (pkg.noarch_arch[rpmfile], `ftptree`, arch)),
        for line in f.readlines(True)[2:]:
            sm.write(line)
        f.close()

    sm.close()

def move_noarch(f, arch, rpmfile, dstpkg):
    if dstpkg.noarch_arch.has_key(rpmfile):
        os.system("rpm -qlp %s | LC_ALL=C sort > %s/files.new" %
                  (incoming_dir + arch + '/' + rpmfile, tmpdir))
        os.system("rpm -qRp %s | LC_ALL=C sort | uniq > %s/reqs.new" %
                  (incoming_dir + arch + '/' + rpmfile, tmpdir))

        files_differ = False
        reqs_differ = False

        if os.system("diff -u %s/%s.filelist %s/files.new > %s/files.diff" %
                     (noarchcachedir, rpmfile, tmpdir, tmpdir)):
            files_differ = True
        if os.system("diff -u %s/%s.reqlist %s/reqs.new > %s/reqs.diff" %
                     (noarchcachedir, rpmfile, tmpdir, tmpdir)):
            reqs_differ = True

        if files_differ or reqs_differ:
            send_noarch_msg(files_differ, reqs_differ, dstpkg, rpmfile, arch)

        rm(incoming_dir + arch + '/' + rpmfile)
    else:
        os.system("rpm -qlp %s > %s/%s.filelist" %
                  (incoming_dir + arch + '/' + rpmfile, noarchcachedir, rpmfile))
        os.system("rpm -qRp %s |sort|uniq > %s/%s.reqlist" %
                  (incoming_dir + arch + '/' + rpmfile, noarchcachedir, rpmfile))
        f.write("file:noarch:%s\ninfo:noarch_arch:%s:%s\n" % (rpmfile, rpmfile, arch))
        mv(incoming_dir + arch + '/' + rpmfile, default_to + 'noarch/RPMS')

# main()
try:
    ftpio.connect('from-incoming-pid-%s' % os.getpid())
except:
    sys.exit(0)

ftptree = BaseFtpTree(cval['default_to'])

if not ftpio.lock(cval['default_to']):
    sys.exit(0)

for uploadinfo in findfiles(incoming_dir + 'SRPMS'):
    content = getcontent(incoming_dir + 'SRPMS/' + uploadinfo)
    if not content:
        continue # Uploading not finished

    pkg = BasePkg(uploadinfo[:-19], content = content)
    srpm = pkg.files['SRPMS'][0]

    if ftptree.has_key(`pkg`):
        ftpio.log("%s already present in %s; removing newer files" % (srpm, ftptree))
        rm(incoming_dir + 'SRPMS/' + srpm)
        f = open(default_to + 'SRPMS/.metadata/' + srpm+'.info', 'a')
        bid = pkg.build.keys()[0]
        build = pkg.build[bid]
        f.write("info:build:%s:requester:%s\ninfo:build:%s:requester_email:%s\n"
                 % (bid, build.requester, bid, build.requester_email))
        f.close()
    else:
        mv(incoming_dir + 'SRPMS/' + srpm, default_to + 'SRPMS/RPMS')

        f = open(default_to + 'SRPMS/.metadata/' + srpm + '.info', 'w')
        f.write(content)
        f.close()

    rm(incoming_dir + 'SRPMS/' + uploadinfo)

for arch in ftp_archs:
    for uploadinfo in findfiles(incoming_dir + arch):
        content = getcontent(incoming_dir + arch + '/' + uploadinfo)
        if not content:
            print "%s not finished uploading" % uploadinfo
            continue # Uploading not finished

        srcpkg = BasePkg(uploadinfo[:-19], content = content)
        srpm = srcpkg.files['SRPMS'][0]

        if not ftptree.has_key(`srcpkg`):
            continue # We require the src.rpm to be present

        dstpkg = BasePkg(`srcpkg`, ftptree)

        if dstpkg.files.has_key(arch):
            ftpio.log("files from %s for arch %s already present in %s; removing newer files" % (`srcpkg`, arch, ftptree))
            for rpmfile in srcpkg.files['ARCH']:
                try:
                    rm(incoming_dir + arch + '/'+rpmfile)
                except OSError, e:
                    l = "Removing %s problem: %s" % (incoming_dir + arch + '/' + rpmfile, e)
                    ftpio.log(l)
                    print l
            rm(incoming_dir + arch + '/' + uploadinfo)
            continue

        f = open(default_to + 'SRPMS/.metadata/' + srpm + '.info', 'a')
        for rpmfile in srcpkg.files['ARCH']:
            if rpmfile[-11:] == '.noarch.rpm' and config.separate_noarch:
                move_noarch(f, arch, rpmfile, dstpkg)
            else:
                f.write("file:%s:%s\n" % (arch, rpmfile))
                srcfile = incoming_dir + arch + '/' + rpmfile

                if is_debuginfo(rpmfile):
                    dstfile = default_to + arch + '/debuginfo'
                else:
                    dstfile = default_to + arch + '/RPMS'

                try:
                    mv(srcfile, dstfile)
                except OSError, e:
                    l = "Moving %s to %s problem: %s" % (srcfile, dstfile, e)
                    ftpio.log(l)
                    print l
        f.close()

        rm(incoming_dir + arch + '/' + uploadinfo)

ftpio.unlock(cval['default_to'])
Commit	Line	Data
	1	#!/usr/bin/env python
	2	# vi: encoding=utf-8 ts=8 sts=4 sw=4 et
	3
	4	import sys, os, stat, time
	5	sys.path.insert(0, os.environ['HOME']+'/pld-ftp-admin/modules')
	6	from config import incoming_dir, default_to, ftp_archs
	7	from config import value as cval
	8	import config
	9	from common import noarchcachedir, tmpdir, fileexists
	10	from baseftptree import BaseFtpTree, BasePkg
	11	from ftptree import FtpTree, Pkg
	12	import ftpio
	13
	14	def rm(file):
	15	os.remove(file)
	16
	17	def mv(src, dst):
	18	os.rename(src, dst + '/' + src.split('/')[-1])
	19
	20	# duplicate code in ftptree.py
	21	def is_debuginfo(nvr):
	22	"""
	23	returns true if NVR is debuginfo package and separate debuginfo is enabled
	24	"""
	25	if not config.separate_debuginfo:
	26	return False
	27	pkg = nvr.split('-')[:-2]
	28	return pkg[-1] == 'debuginfo'
	29
	30	def findfiles(dir):
	31	def filterinfos(x):
	32	if x[-11:] == '.uploadinfo':
	33	return True
	34	else:
	35	return False
	36	return filter(filterinfos, os.listdir(dir))
	37
	38	def getcontent(file):
	39	f = open(file, 'r')
	40	content = f.read()
	41	f.close()
	42	if not content[-5:] == '\nEND\n':
	43	return None
	44	else:
	45	return content[:-4]
	46
	47	def send_noarch_msg(files_differ, reqs_differ, pkg, rpmfile, arch):
	48	req_email=pkg.build[pkg.lastbid].requester_email
	49	req_bid=pkg.lastbid
	50	cc_list=[]
	51	if 'logs_list' in cval:
	52	cc_list.append(cval['logs_list'])
	53	m_subject="Noarch error: %s files differ among builders" % rpmfile
	54	bids=pkg.build.keys()
	55	if len(bids)>1:
	56	for bid in bids:
	57	newcc=pkg.build[bid].requester_email
	58	if req_email!=newcc and newcc not in cc_list:
	59	cc_list.append(newcc)
	60
	61	msg="From: %s\nTo: %s\n" % (cval['from_field'], req_email)
	62	if cc_list:
	63	msg=msg+"Cc: %s\n" % ", ".join(cc_list)
	64	msg=msg+"""X-PLD-Builder: %s
	65	References: <%s@pld.src.builder>
	66	In-Reply-To: <%s@pld.src.builder>
	67	Subject: %s
	68
	69	""" % (cval['xpldbuilder'], req_bid, req_bid, m_subject)
	70
	71	sm = os.popen("/usr/sbin/sendmail -t", "w")
	72
	73	sm.write(msg)
	74
	75	if files_differ:
	76	f=open("%s/files.diff" % tmpdir, 'r')
	77	sm.write("Difference between %s (currently in %s) and %s FILES\n" %
	78	(pkg.noarch_arch[rpmfile], `ftptree`, arch)),
	79	for line in f.readlines(True)[2:]:
	80	sm.write(line)
	81	f.close()
	82
	83	sm.write('\n')
	84
	85	if reqs_differ:
	86	f=open("%s/reqs.diff" % tmpdir, 'r')
	87	sm.write("Difference between %s (currently in %s) and %s REQS\n" %
	88	(pkg.noarch_arch[rpmfile], `ftptree`, arch)),
	89	for line in f.readlines(True)[2:]:
	90	sm.write(line)
	91	f.close()
	92
	93	sm.close()
	94
	95	def move_noarch(f, arch, rpmfile, dstpkg):
	96	if dstpkg.noarch_arch.has_key(rpmfile):
	97	os.system("rpm -qlp %s \| LC_ALL=C sort > %s/files.new" %
	98	(incoming_dir + arch + '/' + rpmfile, tmpdir))
	99	os.system("rpm -qRp %s \| LC_ALL=C sort \| uniq > %s/reqs.new" %
	100	(incoming_dir + arch + '/' + rpmfile, tmpdir))
	101
	102	files_differ = False
	103	reqs_differ = False
	104
	105	if os.system("diff -u %s/%s.filelist %s/files.new > %s/files.diff" %
	106	(noarchcachedir, rpmfile, tmpdir, tmpdir)):
	107	files_differ = True
	108	if os.system("diff -u %s/%s.reqlist %s/reqs.new > %s/reqs.diff" %
	109	(noarchcachedir, rpmfile, tmpdir, tmpdir)):
	110	reqs_differ = True
	111
	112	if files_differ or reqs_differ:
	113	send_noarch_msg(files_differ, reqs_differ, dstpkg, rpmfile, arch)
	114
	115	rm(incoming_dir + arch + '/' + rpmfile)
	116	else:
	117	os.system("rpm -qlp %s > %s/%s.filelist" %
	118	(incoming_dir + arch + '/' + rpmfile, noarchcachedir, rpmfile))
	119	os.system("rpm -qRp %s \|sort\|uniq > %s/%s.reqlist" %
	120	(incoming_dir + arch + '/' + rpmfile, noarchcachedir, rpmfile))
	121	f.write("file:noarch:%s\ninfo:noarch_arch:%s:%s\n" % (rpmfile, rpmfile, arch))
	122	mv(incoming_dir + arch + '/' + rpmfile, default_to + 'noarch/RPMS')
	123
	124	# main()
	125	try:
	126	ftpio.connect('from-incoming-pid-%s' % os.getpid())
	127	except:
	128	sys.exit(0)
	129
	130	ftptree = BaseFtpTree(cval['default_to'])
	131
	132	if not ftpio.lock(cval['default_to']):
	133	sys.exit(0)
	134
	135	for uploadinfo in findfiles(incoming_dir + 'SRPMS'):
	136	content = getcontent(incoming_dir + 'SRPMS/' + uploadinfo)
	137	if not content:
	138	continue # Uploading not finished
	139
	140	pkg = BasePkg(uploadinfo[:-19], content = content)
	141	srpm = pkg.files['SRPMS'][0]
	142
	143	if ftptree.has_key(`pkg`):
	144	ftpio.log("%s already present in %s; removing newer files" % (srpm, ftptree))
	145	rm(incoming_dir + 'SRPMS/' + srpm)
	146	f = open(default_to + 'SRPMS/.metadata/' + srpm+'.info', 'a')
	147	bid = pkg.build.keys()[0]
	148	build = pkg.build[bid]
	149	f.write("info:build:%s:requester:%s\ninfo:build:%s:requester_email:%s\n"
	150	% (bid, build.requester, bid, build.requester_email))
	151	f.close()
	152	else:
	153	mv(incoming_dir + 'SRPMS/' + srpm, default_to + 'SRPMS/RPMS')
	154
	155	f = open(default_to + 'SRPMS/.metadata/' + srpm + '.info', 'w')
	156	f.write(content)
	157	f.close()
	158
	159	rm(incoming_dir + 'SRPMS/' + uploadinfo)
	160
	161	for arch in ftp_archs:
	162	for uploadinfo in findfiles(incoming_dir + arch):
	163	content = getcontent(incoming_dir + arch + '/' + uploadinfo)
	164	if not content:
	165	print "%s not finished uploading" % uploadinfo
	166	continue # Uploading not finished
	167
	168	srcpkg = BasePkg(uploadinfo[:-19], content = content)
	169	srpm = srcpkg.files['SRPMS'][0]
	170
	171	if not ftptree.has_key(`srcpkg`):
	172	continue # We require the src.rpm to be present
	173
	174	dstpkg = BasePkg(`srcpkg`, ftptree)
	175
	176	if dstpkg.files.has_key(arch):
	177	ftpio.log("files from %s for arch %s already present in %s; removing newer files" % (`srcpkg`, arch, ftptree))
	178	for rpmfile in srcpkg.files['ARCH']:
	179	try:
	180	rm(incoming_dir + arch + '/'+rpmfile)
	181	except OSError, e:
	182	l = "Removing %s problem: %s" % (incoming_dir + arch + '/' + rpmfile, e)
	183	ftpio.log(l)
	184	print l
	185	rm(incoming_dir + arch + '/' + uploadinfo)
	186	continue
	187
	188	f = open(default_to + 'SRPMS/.metadata/' + srpm + '.info', 'a')
	189	for rpmfile in srcpkg.files['ARCH']:
	190	if rpmfile[-11:] == '.noarch.rpm' and config.separate_noarch:
	191	move_noarch(f, arch, rpmfile, dstpkg)
	192	else:
	193	f.write("file:%s:%s\n" % (arch, rpmfile))
	194	srcfile = incoming_dir + arch + '/' + rpmfile
	195
	196	if is_debuginfo(rpmfile):
	197	dstfile = default_to + arch + '/debuginfo'
	198	else:
	199	dstfile = default_to + arch + '/RPMS'
	200
	201	try:
	202	mv(srcfile, dstfile)
	203	except OSError, e:
	204	l = "Moving %s to %s problem: %s" % (srcfile, dstfile, e)
	205	ftpio.log(l)
	206	print l
	207	f.close()
	208
	209	rm(incoming_dir + arch + '/' + uploadinfo)
	210
	211	ftpio.unlock(cval['default_to'])