Don't look .infofiles that are younger that 30 seconds.

[projects/pld-ftp-admin.git] / bin / pfa-from-incoming

#!/usr/bin/env python
# vi: encoding=utf-8 ts=8 sts=4 sw=4 et

import sys, os, stat, time
sys.path.insert(0, os.environ['HOME']+'/pld-ftp-admin/modules')
from config import incoming_dir, default_to, ftp_archs
import config
from common import noarchcachedir, tmpdir, fileexists
from baseftptree import BaseFtpTree, BasePkg
cval=config.value
import ftpio

try:
    ftpio.connect('from-incoming-pid-%s' % os.getpid())
except:
    sys.exit(0)

def rm(file):
    os.remove(file)
    #print 'rm: '+file

def mv(src, dst):
    os.rename(src, dst+'/'+src.split('/')[-1])
    #print "mv: %s %s" % (src, dst+'/'+src.split('/')[-1])

def findfiles(dir):
    def filterinfos(x):
        if x[-11:]=='.uploadinfo':
            mtime = os.stat(dir + '/' + x)[ST_MTIME]
            now = time.time()
            # don't look into uploadinfo files for 30 seconds after scp
            if now - mtime > 30:
                return True
            else:
                return False
        else:
            return False
    return filter(filterinfos, os.listdir(dir))

def getcontent(file):
    f=open(file, 'r')
    content=f.read()
    f.close()
    if not content[-5:]=='\nEND\n':
        return None
    else:
        return content[:-4]

ftptree=BaseFtpTree(cval['default_to'])

if not ftpio.lock(cval['default_to']):
    sys.exit(0)

for uploadinfo in findfiles(incoming_dir+'SRPMS'):
    content=getcontent(incoming_dir+'SRPMS/'+uploadinfo)
    if not content:
        continue # Uploading not finished

    pkg=BasePkg(uploadinfo[:-19], content=content)
    srpm=pkg.files['SRPMS'][0]

    if ftptree.has_key(`pkg`):
        ftpio.log("%s already present in %s; removing newer files" %
            (srpm, ftptree))
        rm(incoming_dir+'SRPMS/'+srpm)
        f=open(default_to+'SRPMS/.metadata/'+srpm+'.info', 'a')
        bid=pkg.build.keys()[0]
        build=pkg.build[bid]
        f.write("info:build:%s:requester:%s\ninfo:build:%s:requester_email:%s\n"
                 % (bid, build.requester, bid, build.requester_email))
        f.close()
    else:
        mv(incoming_dir+'SRPMS/'+srpm, default_to+'SRPMS/RPMS')
 
        f=open(default_to+'SRPMS/.metadata/'+srpm+'.info', 'w')
        f.write(content)
        f.close()

    rm(incoming_dir+'SRPMS/'+uploadinfo)

def send_noarch_msg(files_differ, reqs_differ, pkg, rpm, arch):
    req_email=pkg.build[pkg.lastbid].requester_email
    req_bid=pkg.lastbid
    cc_list=[]
    if 'logs_list' in cval:
        cc_list.append(cval['logs_list'])
    m_subject="Noarch error: %s files differ among builders" % rpm
    bids=pkg.build.keys()
    if len(bids)>1:
        for bid in bids:
            newcc=pkg.build[bid].requester_email
            if req_email!=newcc and newcc not in cc_list:
                cc_list.append(newcc)

    msg="From: %s\nTo: %s\n" % (cval['from_field'], req_email)
    if cc_list:
        msg=msg+"Cc: %s\n" % ", ".join(cc_list)
    msg=msg+"""X-PLD-Builder: %s
References: <%s@pld.src.builder>
In-Reply-To: <%s@pld.src.builder>
Subject: %s

""" % (cval['xpldbuilder'], req_bid, req_bid, m_subject)

    sm = os.popen("/usr/sbin/sendmail -t", "w")
    
    sm.write(msg)

    if files_differ:
        f=open("%s/files.diff" % tmpdir, 'r')
        sm.write("Difference between %s (currently in %s) and %s FILES\n" %
              (pkg.noarch_arch[rpm], `ftptree`, arch)),
        for line in f.readlines(True)[2:]:
            sm.write(line)
        f.close()

    sm.write('\n')

    if reqs_differ:
        f=open("%s/reqs.diff" % tmpdir, 'r')
        sm.write("Difference between %s (currently in %s) and %s REQS\n" %
              (pkg.noarch_arch[rpm], `ftptree`, arch)),
        for line in f.readlines(True)[2:]:
            sm.write(line)
        f.close()

    sm.close()

   

def move_noarch(f, arch, rpm, dstpkg):
    if dstpkg.noarch_arch.has_key(rpm):
        os.system("rpm -qlp %s > %s/files.new" %
                  (incoming_dir+arch+'/'+rpm, tmpdir))
        os.system("rpm -qRp %s |sort|uniq > %s/reqs.new" %
                  (incoming_dir+arch+'/'+rpm, tmpdir))

        files_differ=False
        reqs_differ=False

        if os.system("diff -u %s/%s.filelist %s/files.new > %s/files.diff" %
                     (noarchcachedir, rpm, tmpdir, tmpdir)):
            files_differ=True
        if os.system("diff -u %s/%s.reqlist %s/reqs.new > %s/reqs.diff" %
                     (noarchcachedir, rpm, tmpdir, tmpdir)):
            reqs_differ=True

        if files_differ or reqs_differ:
            send_noarch_msg(files_differ, reqs_differ, dstpkg, rpm, arch)

        rm(incoming_dir+arch+'/'+rpm)
    else:
        os.system("rpm -qlp %s > %s/%s.filelist" %
                  (incoming_dir+arch+'/'+rpm, noarchcachedir, rpm))
        os.system("rpm -qRp %s |sort|uniq > %s/%s.reqlist" % 
                  (incoming_dir+arch+'/'+rpm, noarchcachedir, rpm))
        f.write("file:noarch:%s\ninfo:noarch_arch:%s:%s\n" % (rpm, rpm, arch))
        mv(incoming_dir+arch+'/'+rpm, default_to+'noarch/RPMS')

for arch in ftp_archs:
    for uploadinfo in findfiles(incoming_dir+arch):
        content=getcontent(incoming_dir+arch+'/'+uploadinfo)
        if not content:
            continue # Uploading not finished

        srcpkg=BasePkg(uploadinfo[:-19], content=content)
        srpm=srcpkg.files['SRPMS'][0]

        if not ftptree.has_key(`srcpkg`):
            continue # We require the src.rpm to be present

        dstpkg=BasePkg(`srcpkg`, ftptree)

        if dstpkg.files.has_key(arch):
            ftpio.log("files from %s for arch %s already present in %s; removing newer files" % (`srcpkg`, arch, ftptree))
            for rpm in srcpkg.files['ARCH']:
                try:
                    rm(incoming_dir+arch+'/'+rpm)
                except OSError, e:
                    l = "Removing %s problem: %s" % (incoming_dir+arch+'/'+rpm, e)
                    ftpio.log(l)
                    print l
            rm(incoming_dir+arch+'/'+uploadinfo)
            continue

        f=open(default_to+'SRPMS/.metadata/'+srpm+'.info', 'a')
        for rpm in srcpkg.files['ARCH']:
            if rpm[-11:]=='.noarch.rpm' and config.separate_noarch:
                move_noarch(f, arch, rpm, dstpkg)
            else:
                f.write("file:%s:%s\n" % (arch, rpm))
                mv(incoming_dir+arch+'/'+rpm, default_to+arch+'/RPMS')
        f.close()

        rm(incoming_dir+arch+'/'+uploadinfo)

ftpio.unlock(cval['default_to'])