bin/pfa-from-incoming

   1 #!/usr/bin/env python
   2 # vi: encoding=utf-8 ts=8 sts=4 sw=4 et
   3
   4 import sys, os, stat, time
   5 sys.path.insert(0, os.environ['HOME']+'/pld-ftp-admin/modules')
   6 from config import incoming_dir, default_to, ftp_archs
   7 import config
   8 from common import noarchcachedir, tmpdir, fileexists
   9 from baseftptree import BaseFtpTree, BasePkg
  10 cval=config.value
  11 import ftpio
  12
  13 try:
  14     ftpio.connect('from-incoming-pid-%s' % os.getpid())
  15 except:
  16     sys.exit(0)
  17
  18 def rm(file):
  19     os.remove(file)
  20     #print 'rm: '+file
  21
  22 def mv(src, dst):
  23     os.rename(src, dst+'/'+src.split('/')[-1])
  24     #print "mv: %s %s" % (src, dst+'/'+src.split('/')[-1])
  25
  26 def findfiles(dir):
  27     def filterinfos(x):
  28         if x[-11:]=='.uploadinfo':
  29             mtime = os.stat(dir + '/' + x)[stat.ST_MTIME]
  30             now = time.time()
  31             # don't look into uploadinfo files for 30 seconds after scp
  32             # note -- most likely this doesn't achieve anything
  33             if now - mtime > 30:
  34                 return True
  35             else:
  36                 return False
  37         else:
  38             return False
  39     return filter(filterinfos, os.listdir(dir))
  40
  41 def getcontent(file):
  42     f=open(file, 'r')
  43     content=f.read()
  44     f.close()
  45     if not content[-5:]=='\nEND\n':
  46         return None
  47     else:
  48         return content[:-4]
  49
  50 ftptree=BaseFtpTree(cval['default_to'])
  51
  52 if not ftpio.lock(cval['default_to']):
  53     sys.exit(0)
  54
  55 for uploadinfo in findfiles(incoming_dir+'SRPMS'):
  56     content=getcontent(incoming_dir+'SRPMS/'+uploadinfo)
  57     if not content:
  58         continue # Uploading not finished
  59
  60     pkg=BasePkg(uploadinfo[:-19], content=content)
  61     srpm=pkg.files['SRPMS'][0]
  62
  63     if ftptree.has_key(`pkg`):
  64         ftpio.log("%s already present in %s; removing newer files" %
  65             (srpm, ftptree))
  66         rm(incoming_dir+'SRPMS/'+srpm)
  67         f=open(default_to+'SRPMS/.metadata/'+srpm+'.info', 'a')
  68         bid=pkg.build.keys()[0]
  69         build=pkg.build[bid]
  70         f.write("info:build:%s:requester:%s\ninfo:build:%s:requester_email:%s\n"
  71                  % (bid, build.requester, bid, build.requester_email))
  72         f.close()
  73     else:
  74         mv(incoming_dir+'SRPMS/'+srpm, default_to+'SRPMS/RPMS')
  75
  76         f=open(default_to+'SRPMS/.metadata/'+srpm+'.info', 'w')
  77         f.write(content)
  78         f.close()
  79
  80     rm(incoming_dir+'SRPMS/'+uploadinfo)
  81
  82 def send_noarch_msg(files_differ, reqs_differ, pkg, rpm, arch):
  83     req_email=pkg.build[pkg.lastbid].requester_email
  84     req_bid=pkg.lastbid
  85     cc_list=[]
  86     if 'logs_list' in cval:
  87         cc_list.append(cval['logs_list'])
  88     m_subject="Noarch error: %s files differ among builders" % rpm
  89     bids=pkg.build.keys()
  90     if len(bids)>1:
  91         for bid in bids:
  92             newcc=pkg.build[bid].requester_email
  93             if req_email!=newcc and newcc not in cc_list:
  94                 cc_list.append(newcc)
  95
  96     msg="From: %s\nTo: %s\n" % (cval['from_field'], req_email)
  97     if cc_list:
  98         msg=msg+"Cc: %s\n" % ", ".join(cc_list)
  99     msg=msg+"""X-PLD-Builder: %s
 100 References: <%s@pld.src.builder>
 101 In-Reply-To: <%s@pld.src.builder>
 102 Subject: %s
 103
 104 """ % (cval['xpldbuilder'], req_bid, req_bid, m_subject)
 105
 106     sm = os.popen("/usr/sbin/sendmail -t", "w")
 107
 108     sm.write(msg)
 109
 110     if files_differ:
 111         f=open("%s/files.diff" % tmpdir, 'r')
 112         sm.write("Difference between %s (currently in %s) and %s FILES\n" %
 113               (pkg.noarch_arch[rpm], `ftptree`, arch)),
 114         for line in f.readlines(True)[2:]:
 115             sm.write(line)
 116         f.close()
 117
 118     sm.write('\n')
 119
 120     if reqs_differ:
 121         f=open("%s/reqs.diff" % tmpdir, 'r')
 122         sm.write("Difference between %s (currently in %s) and %s REQS\n" %
 123               (pkg.noarch_arch[rpm], `ftptree`, arch)),
 124         for line in f.readlines(True)[2:]:
 125             sm.write(line)
 126         f.close()
 127
 128     sm.close()
 129
 130
 131
 132 def move_noarch(f, arch, rpm, dstpkg):
 133     if dstpkg.noarch_arch.has_key(rpm):
 134         os.system("rpm -qlp %s > %s/files.new" %
 135                   (incoming_dir+arch+'/'+rpm, tmpdir))
 136         os.system("rpm -qRp %s |sort|uniq > %s/reqs.new" %
 137                   (incoming_dir+arch+'/'+rpm, tmpdir))
 138
 139         files_differ=False
 140         reqs_differ=False
 141
 142         if os.system("diff -u %s/%s.filelist %s/files.new > %s/files.diff" %
 143                      (noarchcachedir, rpm, tmpdir, tmpdir)):
 144             files_differ=True
 145         if os.system("diff -u %s/%s.reqlist %s/reqs.new > %s/reqs.diff" %
 146                      (noarchcachedir, rpm, tmpdir, tmpdir)):
 147             reqs_differ=True
 148
 149         if files_differ or reqs_differ:
 150             send_noarch_msg(files_differ, reqs_differ, dstpkg, rpm, arch)
 151
 152         rm(incoming_dir+arch+'/'+rpm)
 153     else:
 154         os.system("rpm -qlp %s > %s/%s.filelist" %
 155                   (incoming_dir+arch+'/'+rpm, noarchcachedir, rpm))
 156         os.system("rpm -qRp %s |sort|uniq > %s/%s.reqlist" %
 157                   (incoming_dir+arch+'/'+rpm, noarchcachedir, rpm))
 158         f.write("file:noarch:%s\ninfo:noarch_arch:%s:%s\n" % (rpm, rpm, arch))
 159         mv(incoming_dir+arch+'/'+rpm, default_to+'noarch/RPMS')
 160
 161 for arch in ftp_archs:
 162     for uploadinfo in findfiles(incoming_dir+arch):
 163         content=getcontent(incoming_dir+arch+'/'+uploadinfo)
 164         if not content:
 165             continue # Uploading not finished
 166
 167         srcpkg=BasePkg(uploadinfo[:-19], content=content)
 168         srpm=srcpkg.files['SRPMS'][0]
 169
 170         if not ftptree.has_key(`srcpkg`):
 171             continue # We require the src.rpm to be present
 172
 173         dstpkg=BasePkg(`srcpkg`, ftptree)
 174
 175         if dstpkg.files.has_key(arch):
 176             ftpio.log("files from %s for arch %s already present in %s; removing newer files" % (`srcpkg`, arch, ftptree))
 177             for rpm in srcpkg.files['ARCH']:
 178                 try:
 179                     rm(incoming_dir+arch+'/'+rpm)
 180                 except OSError, e:
 181                     l = "Removing %s problem: %s" % (incoming_dir+arch+'/'+rpm, e)
 182                     ftpio.log(l)
 183                     print l
 184             rm(incoming_dir+arch+'/'+uploadinfo)
 185             continue
 186
 187         f=open(default_to+'SRPMS/.metadata/'+srpm+'.info', 'a')
 188         for rpm in srcpkg.files['ARCH']:
 189             if rpm[-11:]=='.noarch.rpm' and config.separate_noarch:
 190                 move_noarch(f, arch, rpm, dstpkg)
 191             else:
 192                 f.write("file:%s:%s\n" % (arch, rpm))
 193                 mv(incoming_dir+arch+'/'+rpm, default_to+arch+'/RPMS')
 194         f.close()
 195
 196         rm(incoming_dir+arch+'/'+uploadinfo)
 197
 198 ftpio.unlock(cval['default_to'])
 199