]> git.pld-linux.org Git - projects/pld-ftp-admin.git/blame - bin/pfa-from-incoming
number items
[projects/pld-ftp-admin.git] / bin / pfa-from-incoming
CommitLineData
38b2b764
MM
1#!/usr/bin/env python
2# vi: encoding=utf-8 ts=8 sts=4 sw=4 et
3
c77db9e7 4import sys, os, stat, time, re
38b2b764
MM
5sys.path.insert(0, os.environ['HOME']+'/pld-ftp-admin/modules')
6from config import incoming_dir, default_to, ftp_archs
8e089d4f 7from config import value as cval
38b2b764 8import config
acd976ec 9from common import noarchcachedir, tmpdir, fileexists
bb2cb325 10from baseftptree import BaseFtpTree, BasePkg
8e089d4f 11from ftptree import FtpTree, Pkg
0a108b7f 12import ftpio
38b2b764 13
fc574b00
AM
14os.umask(022)
15
38b2b764 16def rm(file):
18c8e935 17 os.remove(file)
38b2b764
MM
18
19def mv(src, dst):
8911f226 20 os.rename(src, dst + '/' + src.split('/')[-1])
38b2b764 21
54ff0049
ER
22# duplicate code in ftptree.py
23def is_debuginfo(nvr):
24 """
25 returns true if NVR is debuginfo package and separate debuginfo is enabled
26 """
27 if not config.separate_debuginfo:
28 return False
29 pkg = nvr.split('-')[:-2]
30 return pkg[-1] == 'debuginfo'
31
38b2b764
MM
32def findfiles(dir):
33 def filterinfos(x):
8911f226 34 if x[-11:] == '.uploadinfo':
76dc40c3 35 return True
38b2b764
MM
36 else:
37 return False
38 return filter(filterinfos, os.listdir(dir))
39
38b2b764 40def getcontent(file):
8911f226
ER
41 f = open(file, 'r')
42 content = f.read()
38b2b764 43 f.close()
8911f226 44 if not content[-5:] == '\nEND\n':
38b2b764
MM
45 return None
46 else:
47 return content[:-4]
48
8e089d4f 49def send_noarch_msg(files_differ, reqs_differ, pkg, rpmfile, arch):
200470c5
MM
50 req_email=pkg.build[pkg.lastbid].requester_email
51 req_bid=pkg.lastbid
52 cc_list=[]
0fa16f20
MM
53 if 'logs_list' in cval:
54 cc_list.append(cval['logs_list'])
8e089d4f 55 m_subject="Noarch error: %s files differ among builders" % rpmfile
200470c5
MM
56 bids=pkg.build.keys()
57 if len(bids)>1:
58 for bid in bids:
59 newcc=pkg.build[bid].requester_email
60 if req_email!=newcc and newcc not in cc_list:
61 cc_list.append(newcc)
62
63 msg="From: %s\nTo: %s\n" % (cval['from_field'], req_email)
64 if cc_list:
65 msg=msg+"Cc: %s\n" % ", ".join(cc_list)
66 msg=msg+"""X-PLD-Builder: %s
67References: <%s@pld.src.builder>
68In-Reply-To: <%s@pld.src.builder>
69Subject: %s
70
71""" % (cval['xpldbuilder'], req_bid, req_bid, m_subject)
72
73 sm = os.popen("/usr/sbin/sendmail -t", "w")
8e089d4f 74
200470c5
MM
75 sm.write(msg)
76
77 if files_differ:
78 f=open("%s/files.diff" % tmpdir, 'r')
79 sm.write("Difference between %s (currently in %s) and %s FILES\n" %
8e089d4f 80 (pkg.noarch_arch[rpmfile], `ftptree`, arch)),
200470c5
MM
81 for line in f.readlines(True)[2:]:
82 sm.write(line)
83 f.close()
84
85 sm.write('\n')
86
87 if reqs_differ:
88 f=open("%s/reqs.diff" % tmpdir, 'r')
89 sm.write("Difference between %s (currently in %s) and %s REQS\n" %
8e089d4f 90 (pkg.noarch_arch[rpmfile], `ftptree`, arch)),
200470c5
MM
91 for line in f.readlines(True)[2:]:
92 sm.write(line)
93 f.close()
94
95 sm.close()
96
8e089d4f
ER
97def move_noarch(f, arch, rpmfile, dstpkg):
98 if dstpkg.noarch_arch.has_key(rpmfile):
a999d554 99 os.system("LC_ALL=C rpm -qlp %s | LC_ALL=C sort > %s/files.new" %
8e089d4f 100 (incoming_dir + arch + '/' + rpmfile, tmpdir))
ca2e224a 101 os.system("rpm -qRp %s | LC_ALL=C sort | LC_ALL=C uniq > %s/reqs.new" %
8e089d4f 102 (incoming_dir + arch + '/' + rpmfile, tmpdir))
bb2cb325 103
8911f226
ER
104 files_differ = False
105 reqs_differ = False
bb2cb325
MM
106
107 if os.system("diff -u %s/%s.filelist %s/files.new > %s/files.diff" %
8e089d4f 108 (noarchcachedir, rpmfile, tmpdir, tmpdir)):
8911f226 109 files_differ = True
bb2cb325 110 if os.system("diff -u %s/%s.reqlist %s/reqs.new > %s/reqs.diff" %
8e089d4f 111 (noarchcachedir, rpmfile, tmpdir, tmpdir)):
8911f226 112 reqs_differ = True
bb2cb325
MM
113
114 if files_differ or reqs_differ:
8e089d4f 115 send_noarch_msg(files_differ, reqs_differ, dstpkg, rpmfile, arch)
bb2cb325 116
8e089d4f 117 rm(incoming_dir + arch + '/' + rpmfile)
bb2cb325 118 else:
ca2e224a 119 os.system("LC_ALL=C rpm -qlp %s | LC_ALL=C sort > %s/%s.filelist" %
8e089d4f 120 (incoming_dir + arch + '/' + rpmfile, noarchcachedir, rpmfile))
ca2e224a 121 os.system("rpm -qRp %s | LC_ALL=C sort | LC_ALL=C uniq > %s/%s.reqlist" %
8e089d4f 122 (incoming_dir + arch + '/' + rpmfile, noarchcachedir, rpmfile))
c0d14f9d
JR
123 if not dstpkg.files.has_key(arch):
124 f.write("file:noarch:%s\ninfo:noarch_arch:%s:%s\n" % (rpmfile, rpmfile, arch))
8e089d4f 125 mv(incoming_dir + arch + '/' + rpmfile, default_to + 'noarch/RPMS')
38b2b764 126
c77db9e7
JR
127def send_vr_msg(snvr, anvr, pkg, arch):
128 req_email=pkg.build[pkg.lastbid].requester_email
129 req_bid=pkg.lastbid
130 cc_list=[]
131 if 'logs_list' in cval:
132 cc_list.append(cval['logs_list'])
133 m_subject="NVR error: %s version or relese differ among subpackages" % snvr[0]
134 bids=pkg.build.keys()
135 if len(bids)>1:
136 for bid in bids:
137 newcc=pkg.build[bid].requester_email
138 if req_email!=newcc and newcc not in cc_list:
139 cc_list.append(newcc)
140
141 msg="From: %s\nTo: %s\n" % (cval['from_field'], req_email)
142 if cc_list:
143 msg=msg+"Cc: %s\n" % ", ".join(cc_list)
144 msg=msg+"""X-PLD-Builder: %s
145References: <%s@pld.src.builder>
146In-Reply-To: <%s@pld.src.builder>
147Subject: %s
148
149""" % (cval['xpldbuilder'], req_bid, req_bid, m_subject)
150
151 sm = os.popen("/usr/sbin/sendmail -t", "w")
152
153 sm.write(msg)
154
155 sm.write("Difference between %s SRPM (currently in %s) and %s RPM NVR:\n\n" %
156 (snvr[0], `ftptree`, arch)),
c82d0ed4
JR
157 sm.write("Expected (%s):\nV: %s\nR: %s\n\n" % snvr)
158 sm.write("RPM:\nN: %s\nV: %s\nR: %s\n" % anvr)
c77db9e7
JR
159 sm.write('\n')
160
161 sm.close()
162
e4b21e2b
ER
163# main()
164try:
165 ftpio.connect('from-incoming-pid-%s' % os.getpid())
166except:
65017da5
ER
167 print "Can't get ftpiod connection"
168 sys.exit(1)
e4b21e2b
ER
169
170ftptree = BaseFtpTree(cval['default_to'])
171
172if not ftpio.lock(cval['default_to']):
65017da5
ER
173 print "Can't get lock: %s" % cval['default_to']
174 sys.exit(1)
e4b21e2b 175
7c2f5e4c
AM
176moved_anything = False
177
e4b21e2b
ER
178for uploadinfo in findfiles(incoming_dir + 'SRPMS'):
179 content = getcontent(incoming_dir + 'SRPMS/' + uploadinfo)
180 if not content:
181 continue # Uploading not finished
182
183 pkg = BasePkg(uploadinfo[:-19], content = content)
184 srpm = pkg.files['SRPMS'][0]
185
46d96a92
AM
186 if not os.path.exists(incoming_dir + 'SRPMS/' + srpm):
187 ftpio.log("%s file missing; skipping move until next round" % (srpm))
188 continue
189
e4b21e2b 190 if ftptree.has_key(`pkg`):
99b125ec
JR
191 ftpio.log("%s already present in %s; removing older files" % (srpm, ftptree))
192 rm(default_to + 'SRPMS/RPMS/' + srpm)
e4b21e2b
ER
193 f = open(default_to + 'SRPMS/.metadata/' + srpm+'.info', 'a')
194 bid = pkg.build.keys()[0]
195 build = pkg.build[bid]
196 f.write("info:build:%s:requester:%s\ninfo:build:%s:requester_email:%s\n"
197 % (bid, build.requester, bid, build.requester_email))
198 f.close()
199 else:
e4b21e2b
ER
200 f = open(default_to + 'SRPMS/.metadata/' + srpm + '.info', 'w')
201 f.write(content)
202 f.close()
203
99b125ec 204 mv(incoming_dir + 'SRPMS/' + srpm, default_to + 'SRPMS/RPMS')
e4b21e2b
ER
205 rm(incoming_dir + 'SRPMS/' + uploadinfo)
206
38b2b764 207for arch in ftp_archs:
8e089d4f 208 for uploadinfo in findfiles(incoming_dir + arch):
8911f226 209 content = getcontent(incoming_dir + arch + '/' + uploadinfo)
38b2b764 210 if not content:
2abd3b77 211 ftpio.log("%s not finished uploading" % uploadinfo)
38b2b764
MM
212 continue # Uploading not finished
213
8911f226
ER
214 srcpkg = BasePkg(uploadinfo[:-19], content = content)
215 srpm = srcpkg.files['SRPMS'][0]
38b2b764 216
bb2cb325 217 if not ftptree.has_key(`srcpkg`):
38b2b764
MM
218 continue # We require the src.rpm to be present
219
c77db9e7
JR
220 renvr = re.compile(r'(.*)-(.*)-(.*)\.[^.]*\.rpm')
221 srcnvr = renvr.match(srpm).groups()
222
46d96a92
AM
223 rpmfile_missing = [f for f in srcpkg.files['ARCH'] if not os.path.exists(incoming_dir + arch + '/'+f)]
224 if len(rpmfile_missing):
225 for filem in rpmfile_missing:
226 ftpio.log("%s file missing; skipping move until next round" % (filem))
227 continue
228
8911f226 229 dstpkg = BasePkg(`srcpkg`, ftptree)
38b2b764 230
bb2cb325 231 if dstpkg.files.has_key(arch):
99b125ec
JR
232 ftpio.log("files from %s for arch %s already present in %s; removing older files" % (`srcpkg`, arch, ftptree))
233 for rpmfile in dstpkg.files[arch]:
234 if is_debuginfo(rpmfile):
235 dstfile = default_to + arch + '/debuginfo'
236 else:
237 dstfile = default_to + arch + '/RPMS'
2eb653d6 238 try:
99b125ec 239 rm(dstfile + '/' + rpmfile)
2eb653d6 240 except OSError, e:
99b125ec 241 l = "Removing %s problem: %s" % (dstfile + '/' + rpmfile, e)
2eb653d6
AM
242 ftpio.log(l)
243 print l
38b2b764 244
8911f226 245 f = open(default_to + 'SRPMS/.metadata/' + srpm + '.info', 'a')
8e089d4f 246 for rpmfile in srcpkg.files['ARCH']:
7c2f5e4c 247 moved_anything = True
c77db9e7 248
164d8d15
JR
249# Too much noise, too little use
250# archnvr = renvr.match(rpmfile).groups()
251# if srcnvr[1] != archnvr[1] or srcnvr[2] != archnvr[2]:
252# send_vr_msg(srcnvr, archnvr, dstpkg, arch)
c77db9e7 253
8e089d4f
ER
254 if rpmfile[-11:] == '.noarch.rpm' and config.separate_noarch:
255 move_noarch(f, arch, rpmfile, dstpkg)
bb2cb325 256 else:
c0d14f9d
JR
257 if not dstpkg.files.has_key(arch):
258 f.write("file:%s:%s\n" % (arch, rpmfile))
8e089d4f 259 srcfile = incoming_dir + arch + '/' + rpmfile
8911f226 260
8e089d4f 261 if is_debuginfo(rpmfile):
54ff0049
ER
262 dstfile = default_to + arch + '/debuginfo'
263 else:
264 dstfile = default_to + arch + '/RPMS'
8911f226 265
631d11b9 266 try:
8911f226 267 mv(srcfile, dstfile)
631d11b9 268 except OSError, e:
8911f226 269 l = "Moving %s to %s problem: %s" % (srcfile, dstfile, e)
631d11b9
AM
270 ftpio.log(l)
271 print l
38b2b764
MM
272 f.close()
273
8911f226 274 rm(incoming_dir + arch + '/' + uploadinfo)
38b2b764 275
0a108b7f 276ftpio.unlock(cval['default_to'])
7c2f5e4c
AM
277
278if moved_anything:
a34f0e55 279 os.system("%s/pld-ftp-admin/bin/pfa-genindex --quiet test > /dev/null" % (os.getenv("HOME")))
This page took 0.960938 seconds and 4 git commands to generate.