[projects/pld-ftp-admin.git] / bin / pfa-from-incoming

#!/usr/bin/env python3
# vi: encoding=utf-8 ts=8 sts=4 sw=4 et

from __future__ import print_function

import sys, os, stat, time, re
sys.path.insert(0, os.environ['HOME']+'/pld-ftp-admin/modules')
from config import incoming_dir, default_to, ftp_archs
from config import value as cval
import config
from common import noarchcachedir, tmpdir, fileexists
from baseftptree import BaseFtpTree, BasePkg
from ftptree import FtpTree, Pkg
import ftpio

os.umask(0o022)

def rm(file):
    os.remove(file)

def mv(src, dst):
    os.rename(src, dst + '/' + src.split('/')[-1])

# duplicate code in ftptree.py
def is_debuginfo(nvr):
    """
    returns true if NVR is debuginfo package and separate debuginfo is enabled
    """
    if not config.separate_debuginfo:
        return False
    pkg = nvr.split('-')[:-2]
    return pkg[-1] == 'debuginfo'

def findfiles(dir):
    def filterinfos(x):
        if x[-11:] == '.uploadinfo':
            return True
        else:
            return False
    return filter(filterinfos, os.listdir(dir))

def getcontent(file):
    f = open(file, 'r')
    content = f.read()
    f.close()
    if not content[-5:] == '\nEND\n':
        return None
    else:
        return content[:-4]

def send_noarch_msg(files_differ, reqs_differ, pkg, rpmfile, arch):
    req_email=pkg.build[pkg.lastbid].requester_email
    req_bid=pkg.lastbid
    cc_list=[]
    if 'logs_list' in cval:
        cc_list.append(cval['logs_list'])
    m_subject="Noarch error: %s files differ among builders" % rpmfile
    bids=pkg.build.keys()
    if len(bids)>1:
        for bid in bids:
            newcc=pkg.build[bid].requester_email
            if req_email!=newcc and newcc not in cc_list:
                cc_list.append(newcc)

    msg="From: %s\nTo: %s\n" % (cval['from_field'], req_email)
    if cc_list:
        msg=msg+"Cc: %s\n" % ", ".join(cc_list)
    msg=msg+"""X-PLD-Builder: %s
References: <%s@pld.src.builder>
In-Reply-To: <%s@pld.src.builder>
Subject: %s

""" % (cval['xpldbuilder'], req_bid, req_bid, m_subject)

    sm = os.popen("/usr/sbin/sendmail -t", "w")

    sm.write(msg)

    if files_differ:
        f=open("%s/files.diff" % tmpdir, 'r')
        sm.write("Difference between %s (currently in %s) and %s FILES\n" %
              (pkg.noarch_arch[rpmfile], repr(ftptree), arch)),
        for line in f.readlines(True)[2:]:
            sm.write(line)
        f.close()

    sm.write('\n')

    if reqs_differ:
        f=open("%s/reqs.diff" % tmpdir, 'r')
        sm.write("Difference between %s (currently in %s) and %s REQS\n" %
              (pkg.noarch_arch[rpmfile], repr(ftptree), arch)),
        for line in f.readlines(True)[2:]:
            sm.write(line)
        f.close()

    sm.close()

def move_noarch(f, arch, rpmfile, dstpkg):
    if rpmfile in dstpkg.noarch_arch:
        os.system("LC_ALL=C rpm -qlp %s | LC_ALL=C sort > %s/files.new" %
                  (incoming_dir + arch + '/' + rpmfile, tmpdir))
        os.system("rpm -qRp %s | LC_ALL=C sort | LC_ALL=C uniq > %s/reqs.new" %
                  (incoming_dir + arch + '/' + rpmfile, tmpdir))

        files_differ = False
        reqs_differ = False

        if os.system("diff -u %s/%s.filelist %s/files.new > %s/files.diff" %
                     (noarchcachedir, rpmfile, tmpdir, tmpdir)):
            files_differ = True
        if os.system("diff -u %s/%s.reqlist %s/reqs.new > %s/reqs.diff" %
                     (noarchcachedir, rpmfile, tmpdir, tmpdir)):
            reqs_differ = True

        if files_differ or reqs_differ:
            send_noarch_msg(files_differ, reqs_differ, dstpkg, rpmfile, arch)

        rm(incoming_dir + arch + '/' + rpmfile)
    else:
        os.system("LC_ALL=C rpm -qlp %s | LC_ALL=C sort > %s/%s.filelist" %
                  (incoming_dir + arch + '/' + rpmfile, noarchcachedir, rpmfile))
        os.system("rpm -qRp %s | LC_ALL=C sort | LC_ALL=C uniq > %s/%s.reqlist" %
                  (incoming_dir + arch + '/' + rpmfile, noarchcachedir, rpmfile))
        if arch not in dstpkg.files:
            f.write("file:noarch:%s\ninfo:noarch_arch:%s:%s\n" % (rpmfile, rpmfile, arch))
        mv(incoming_dir + arch + '/' + rpmfile, default_to + 'noarch/RPMS')

def send_vr_msg(snvr, anvr, pkg, arch):
    req_email=pkg.build[pkg.lastbid].requester_email
    req_bid=pkg.lastbid
    cc_list=[]
    if 'logs_list' in cval:
        cc_list.append(cval['logs_list'])
    m_subject="NVR error: %s version or relese differ among subpackages" % snvr[0]
    bids=pkg.build.keys()
    if len(bids)>1:
        for bid in bids:
            newcc=pkg.build[bid].requester_email
            if req_email!=newcc and newcc not in cc_list:
                cc_list.append(newcc)

    msg="From: %s\nTo: %s\n" % (cval['from_field'], req_email)
    if cc_list:
        msg=msg+"Cc: %s\n" % ", ".join(cc_list)
    msg=msg+"""X-PLD-Builder: %s
References: <%s@pld.src.builder>
In-Reply-To: <%s@pld.src.builder>
Subject: %s

""" % (cval['xpldbuilder'], req_bid, req_bid, m_subject)

    sm = os.popen("/usr/sbin/sendmail -t", "w")

    sm.write(msg)

    sm.write("Difference between %s SRPM (currently in %s) and %s RPM NVR:\n\n" %
          (snvr[0], repr(ftptree), arch)),
    sm.write("Expected (%s):\nV: %s\nR: %s\n\n" % snvr)
    sm.write("RPM:\nN: %s\nV: %s\nR: %s\n" % anvr)
    sm.write('\n')

    sm.close()

# main()
try:
    ftpio.connect('from-incoming-pid-%s' % os.getpid())
except:
    print("Can't get ftpiod connection")
    sys.exit(1)

ftptree = BaseFtpTree(cval['default_to'])

if not ftpio.lock(cval['default_to']):
    print("Can't get lock: %s" % cval['default_to'])
    sys.exit(1)

moved_anything = False

for uploadinfo in findfiles(incoming_dir + 'SRPMS'):
    content = getcontent(incoming_dir + 'SRPMS/' + uploadinfo)
    if not content:
        continue # Uploading not finished

    pkg = BasePkg(uploadinfo[:-19], content = content)
    srpm = pkg.files['SRPMS'][0]

    if not os.path.exists(incoming_dir + 'SRPMS/' + srpm):
        ftpio.log("%s file missing; skipping move until next round" % (srpm))
        continue

    if ftptree.has_key(repr(pkg)):
        ftpio.log("%s already present in %s; removing older files" % (srpm, ftptree))
        rm(default_to + 'SRPMS/RPMS/' + srpm)
        f = open(default_to + 'SRPMS/.metadata/' + srpm+'.info', 'a')
        bid = list(pkg.build.keys())[0]
        build = pkg.build[bid]
        f.write("info:build:%s:requester:%s\ninfo:build:%s:requester_email:%s\n"
                 % (bid, build.requester, bid, build.requester_email))
        f.close()
    else:
        f = open(default_to + 'SRPMS/.metadata/' + srpm + '.info', 'w')
        f.write(content)
        f.close()

    mv(incoming_dir + 'SRPMS/' + srpm, default_to + 'SRPMS/RPMS')
    rm(incoming_dir + 'SRPMS/' + uploadinfo)

for arch in ftp_archs:
    for uploadinfo in findfiles(incoming_dir + arch):
        content = getcontent(incoming_dir + arch + '/' + uploadinfo)
        if not content:
            ftpio.log("%s not finished uploading" % uploadinfo)
            continue # Uploading not finished

        srcpkg = BasePkg(uploadinfo[:-19], content = content)
        srpm = srcpkg.files['SRPMS'][0]

        if not ftptree.has_key(repr(srcpkg)):
            continue # We require the src.rpm to be present

        renvr = re.compile(r'(.*)-(.*)-(.*)\.[^.]*\.rpm')
        srcnvr = renvr.match(srpm).groups()

        rpmfile_missing = [f for f in srcpkg.files['ARCH'] if not os.path.exists(incoming_dir + arch + '/'+f)]
        if len(rpmfile_missing):
            for filem in rpmfile_missing:
                ftpio.log("%s file missing; skipping move until next round" % (filem))
            continue

        dstpkg = BasePkg(repr(srcpkg), ftptree)

        if arch in dstpkg.files:
            ftpio.log("files from %s for arch %s already present in %s; removing older files" % (repr(srcpkg), arch, ftptree))
            for rpmfile in dstpkg.files[arch]:
                if is_debuginfo(rpmfile):
                    dstfile = default_to + arch + '/debuginfo'
                else:
                    dstfile = default_to + arch + '/RPMS'
                try:
                    rm(dstfile + '/' + rpmfile)
                except OSError as e:
                    l = "Removing %s problem: %s" % (dstfile + '/' + rpmfile, e)
                    ftpio.log(l)
                    print(l)

        f = open(default_to + 'SRPMS/.metadata/' + srpm + '.info', 'a')
        for rpmfile in srcpkg.files['ARCH']:
            moved_anything = True

# Too much noise, too little use
#            archnvr = renvr.match(rpmfile).groups()
#            if srcnvr[1] != archnvr[1] or srcnvr[2] != archnvr[2]:
#                send_vr_msg(srcnvr, archnvr, dstpkg, arch)

            if rpmfile[-11:] == '.noarch.rpm' and config.separate_noarch:
                move_noarch(f, arch, rpmfile, dstpkg)
            else:
                if arch not in dstpkg.files:
                    f.write("file:%s:%s\n" % (arch, rpmfile))
                srcfile = incoming_dir + arch + '/' + rpmfile

                if is_debuginfo(rpmfile):
                    dstfile = default_to + arch + '/debuginfo'
                else:
                    dstfile = default_to + arch + '/RPMS'

                try:
                    mv(srcfile, dstfile)
                except OSError as e:
                    l = "Moving %s to %s problem: %s" % (srcfile, dstfile, e)
                    ftpio.log(l)
                    print(l)
        f.close()

        rm(incoming_dir + arch + '/' + uploadinfo)

ftpio.unlock(cval['default_to'])

if moved_anything:
    os.system("%s/pld-ftp-admin/bin/pfa-genindex --quiet test > /dev/null" % (os.getenv("HOME")))
Commit	Line	Data
	1	#!/usr/bin/env python3
	2	# vi: encoding=utf-8 ts=8 sts=4 sw=4 et
	3
	4	from __future__ import print_function
	5
	6	import sys, os, stat, time, re
	7	sys.path.insert(0, os.environ['HOME']+'/pld-ftp-admin/modules')
	8	from config import incoming_dir, default_to, ftp_archs
	9	from config import value as cval
	10	import config
	11	from common import noarchcachedir, tmpdir, fileexists
	12	from baseftptree import BaseFtpTree, BasePkg
	13	from ftptree import FtpTree, Pkg
	14	import ftpio
	15
	16	os.umask(0o022)
	17
	18	def rm(file):
	19	os.remove(file)
	20
	21	def mv(src, dst):
	22	os.rename(src, dst + '/' + src.split('/')[-1])
	23
	24	# duplicate code in ftptree.py
	25	def is_debuginfo(nvr):
	26	"""
	27	returns true if NVR is debuginfo package and separate debuginfo is enabled
	28	"""
	29	if not config.separate_debuginfo:
	30	return False
	31	pkg = nvr.split('-')[:-2]
	32	return pkg[-1] == 'debuginfo'
	33
	34	def findfiles(dir):
	35	def filterinfos(x):
	36	if x[-11:] == '.uploadinfo':
	37	return True
	38	else:
	39	return False
	40	return filter(filterinfos, os.listdir(dir))
	41
	42	def getcontent(file):
	43	f = open(file, 'r')
	44	content = f.read()
	45	f.close()
	46	if not content[-5:] == '\nEND\n':
	47	return None
	48	else:
	49	return content[:-4]
	50
	51	def send_noarch_msg(files_differ, reqs_differ, pkg, rpmfile, arch):
	52	req_email=pkg.build[pkg.lastbid].requester_email
	53	req_bid=pkg.lastbid
	54	cc_list=[]
	55	if 'logs_list' in cval:
	56	cc_list.append(cval['logs_list'])
	57	m_subject="Noarch error: %s files differ among builders" % rpmfile
	58	bids=pkg.build.keys()
	59	if len(bids)>1:
	60	for bid in bids:
	61	newcc=pkg.build[bid].requester_email
	62	if req_email!=newcc and newcc not in cc_list:
	63	cc_list.append(newcc)
	64
	65	msg="From: %s\nTo: %s\n" % (cval['from_field'], req_email)
	66	if cc_list:
	67	msg=msg+"Cc: %s\n" % ", ".join(cc_list)
	68	msg=msg+"""X-PLD-Builder: %s
	69	References: <%s@pld.src.builder>
	70	In-Reply-To: <%s@pld.src.builder>
	71	Subject: %s
	72
	73	""" % (cval['xpldbuilder'], req_bid, req_bid, m_subject)
	74
	75	sm = os.popen("/usr/sbin/sendmail -t", "w")
	76
	77	sm.write(msg)
	78
	79	if files_differ:
	80	f=open("%s/files.diff" % tmpdir, 'r')
	81	sm.write("Difference between %s (currently in %s) and %s FILES\n" %
	82	(pkg.noarch_arch[rpmfile], repr(ftptree), arch)),
	83	for line in f.readlines(True)[2:]:
	84	sm.write(line)
	85	f.close()
	86
	87	sm.write('\n')
	88
	89	if reqs_differ:
	90	f=open("%s/reqs.diff" % tmpdir, 'r')
	91	sm.write("Difference between %s (currently in %s) and %s REQS\n" %
	92	(pkg.noarch_arch[rpmfile], repr(ftptree), arch)),
	93	for line in f.readlines(True)[2:]:
	94	sm.write(line)
	95	f.close()
	96
	97	sm.close()
	98
	99	def move_noarch(f, arch, rpmfile, dstpkg):
	100	if rpmfile in dstpkg.noarch_arch:
	101	os.system("LC_ALL=C rpm -qlp %s \| LC_ALL=C sort > %s/files.new" %
	102	(incoming_dir + arch + '/' + rpmfile, tmpdir))
	103	os.system("rpm -qRp %s \| LC_ALL=C sort \| LC_ALL=C uniq > %s/reqs.new" %
	104	(incoming_dir + arch + '/' + rpmfile, tmpdir))
	105
	106	files_differ = False
	107	reqs_differ = False
	108
	109	if os.system("diff -u %s/%s.filelist %s/files.new > %s/files.diff" %
	110	(noarchcachedir, rpmfile, tmpdir, tmpdir)):
	111	files_differ = True
	112	if os.system("diff -u %s/%s.reqlist %s/reqs.new > %s/reqs.diff" %
	113	(noarchcachedir, rpmfile, tmpdir, tmpdir)):
	114	reqs_differ = True
	115
	116	if files_differ or reqs_differ:
	117	send_noarch_msg(files_differ, reqs_differ, dstpkg, rpmfile, arch)
	118
	119	rm(incoming_dir + arch + '/' + rpmfile)
	120	else:
	121	os.system("LC_ALL=C rpm -qlp %s \| LC_ALL=C sort > %s/%s.filelist" %
	122	(incoming_dir + arch + '/' + rpmfile, noarchcachedir, rpmfile))
	123	os.system("rpm -qRp %s \| LC_ALL=C sort \| LC_ALL=C uniq > %s/%s.reqlist" %
	124	(incoming_dir + arch + '/' + rpmfile, noarchcachedir, rpmfile))
	125	if arch not in dstpkg.files:
	126	f.write("file:noarch:%s\ninfo:noarch_arch:%s:%s\n" % (rpmfile, rpmfile, arch))
	127	mv(incoming_dir + arch + '/' + rpmfile, default_to + 'noarch/RPMS')
	128
	129	def send_vr_msg(snvr, anvr, pkg, arch):
	130	req_email=pkg.build[pkg.lastbid].requester_email
	131	req_bid=pkg.lastbid
	132	cc_list=[]
	133	if 'logs_list' in cval:
	134	cc_list.append(cval['logs_list'])
	135	m_subject="NVR error: %s version or relese differ among subpackages" % snvr[0]
	136	bids=pkg.build.keys()
	137	if len(bids)>1:
	138	for bid in bids:
	139	newcc=pkg.build[bid].requester_email
	140	if req_email!=newcc and newcc not in cc_list:
	141	cc_list.append(newcc)
	142
	143	msg="From: %s\nTo: %s\n" % (cval['from_field'], req_email)
	144	if cc_list:
	145	msg=msg+"Cc: %s\n" % ", ".join(cc_list)
	146	msg=msg+"""X-PLD-Builder: %s
	147	References: <%s@pld.src.builder>
	148	In-Reply-To: <%s@pld.src.builder>
	149	Subject: %s
	150
	151	""" % (cval['xpldbuilder'], req_bid, req_bid, m_subject)
	152
	153	sm = os.popen("/usr/sbin/sendmail -t", "w")
	154
	155	sm.write(msg)
	156
	157	sm.write("Difference between %s SRPM (currently in %s) and %s RPM NVR:\n\n" %
	158	(snvr[0], repr(ftptree), arch)),
	159	sm.write("Expected (%s):\nV: %s\nR: %s\n\n" % snvr)
	160	sm.write("RPM:\nN: %s\nV: %s\nR: %s\n" % anvr)
	161	sm.write('\n')
	162
	163	sm.close()
	164
	165	# main()
	166	try:
	167	ftpio.connect('from-incoming-pid-%s' % os.getpid())
	168	except:
	169	print("Can't get ftpiod connection")
	170	sys.exit(1)
	171
	172	ftptree = BaseFtpTree(cval['default_to'])
	173
	174	if not ftpio.lock(cval['default_to']):
	175	print("Can't get lock: %s" % cval['default_to'])
	176	sys.exit(1)
	177
	178	moved_anything = False
	179
	180	for uploadinfo in findfiles(incoming_dir + 'SRPMS'):
	181	content = getcontent(incoming_dir + 'SRPMS/' + uploadinfo)
	182	if not content:
	183	continue # Uploading not finished
	184
	185	pkg = BasePkg(uploadinfo[:-19], content = content)
	186	srpm = pkg.files['SRPMS'][0]
	187
	188	if not os.path.exists(incoming_dir + 'SRPMS/' + srpm):
	189	ftpio.log("%s file missing; skipping move until next round" % (srpm))
	190	continue
	191
	192	if ftptree.has_key(repr(pkg)):
	193	ftpio.log("%s already present in %s; removing older files" % (srpm, ftptree))
	194	rm(default_to + 'SRPMS/RPMS/' + srpm)
	195	f = open(default_to + 'SRPMS/.metadata/' + srpm+'.info', 'a')
	196	bid = list(pkg.build.keys())[0]
	197	build = pkg.build[bid]
	198	f.write("info:build:%s:requester:%s\ninfo:build:%s:requester_email:%s\n"
	199	% (bid, build.requester, bid, build.requester_email))
	200	f.close()
	201	else:
	202	f = open(default_to + 'SRPMS/.metadata/' + srpm + '.info', 'w')
	203	f.write(content)
	204	f.close()
	205
	206	mv(incoming_dir + 'SRPMS/' + srpm, default_to + 'SRPMS/RPMS')
	207	rm(incoming_dir + 'SRPMS/' + uploadinfo)
	208
	209	for arch in ftp_archs:
	210	for uploadinfo in findfiles(incoming_dir + arch):
	211	content = getcontent(incoming_dir + arch + '/' + uploadinfo)
	212	if not content:
	213	ftpio.log("%s not finished uploading" % uploadinfo)
	214	continue # Uploading not finished
	215
	216	srcpkg = BasePkg(uploadinfo[:-19], content = content)
	217	srpm = srcpkg.files['SRPMS'][0]
	218
	219	if not ftptree.has_key(repr(srcpkg)):
	220	continue # We require the src.rpm to be present
	221
	222	renvr = re.compile(r'(.)-(.)-(.)\.[^.]\.rpm')
	223	srcnvr = renvr.match(srpm).groups()
	224
	225	rpmfile_missing = [f for f in srcpkg.files['ARCH'] if not os.path.exists(incoming_dir + arch + '/'+f)]
	226	if len(rpmfile_missing):
	227	for filem in rpmfile_missing:
	228	ftpio.log("%s file missing; skipping move until next round" % (filem))
	229	continue
	230
	231	dstpkg = BasePkg(repr(srcpkg), ftptree)
	232
	233	if arch in dstpkg.files:
	234	ftpio.log("files from %s for arch %s already present in %s; removing older files" % (repr(srcpkg), arch, ftptree))
	235	for rpmfile in dstpkg.files[arch]:
	236	if is_debuginfo(rpmfile):
	237	dstfile = default_to + arch + '/debuginfo'
	238	else:
	239	dstfile = default_to + arch + '/RPMS'
	240	try:
	241	rm(dstfile + '/' + rpmfile)
	242	except OSError as e:
	243	l = "Removing %s problem: %s" % (dstfile + '/' + rpmfile, e)
	244	ftpio.log(l)
	245	print(l)
	246
	247	f = open(default_to + 'SRPMS/.metadata/' + srpm + '.info', 'a')
	248	for rpmfile in srcpkg.files['ARCH']:
	249	moved_anything = True
	250
	251	# Too much noise, too little use
	252	# archnvr = renvr.match(rpmfile).groups()
	253	# if srcnvr[1] != archnvr[1] or srcnvr[2] != archnvr[2]:
	254	# send_vr_msg(srcnvr, archnvr, dstpkg, arch)
	255
	256	if rpmfile[-11:] == '.noarch.rpm' and config.separate_noarch:
	257	move_noarch(f, arch, rpmfile, dstpkg)
	258	else:
	259	if arch not in dstpkg.files:
	260	f.write("file:%s:%s\n" % (arch, rpmfile))
	261	srcfile = incoming_dir + arch + '/' + rpmfile
	262
	263	if is_debuginfo(rpmfile):
	264	dstfile = default_to + arch + '/debuginfo'
	265	else:
	266	dstfile = default_to + arch + '/RPMS'
	267
	268	try:
	269	mv(srcfile, dstfile)
	270	except OSError as e:
	271	l = "Moving %s to %s problem: %s" % (srcfile, dstfile, e)
	272	ftpio.log(l)
	273	print(l)
	274	f.close()
	275
	276	rm(incoming_dir + arch + '/' + uploadinfo)
	277
	278	ftpio.unlock(cval['default_to'])
	279
	280	if moved_anything:
	281	os.system("%s/pld-ftp-admin/bin/pfa-genindex --quiet test > /dev/null" % (os.getenv("HOME")))