]>
Commit | Line | Data |
---|---|---|
38b2b764 MM |
1 | #!/usr/bin/env python |
2 | # vi: encoding=utf-8 ts=8 sts=4 sw=4 et | |
3 | ||
c77db9e7 | 4 | import sys, os, stat, time, re |
38b2b764 MM |
5 | sys.path.insert(0, os.environ['HOME']+'/pld-ftp-admin/modules') |
6 | from config import incoming_dir, default_to, ftp_archs | |
8e089d4f | 7 | from config import value as cval |
38b2b764 | 8 | import config |
acd976ec | 9 | from common import noarchcachedir, tmpdir, fileexists |
bb2cb325 | 10 | from baseftptree import BaseFtpTree, BasePkg |
8e089d4f | 11 | from ftptree import FtpTree, Pkg |
0a108b7f | 12 | import ftpio |
38b2b764 | 13 | |
fc574b00 AM |
14 | os.umask(022) |
15 | ||
38b2b764 | 16 | def rm(file): |
18c8e935 | 17 | os.remove(file) |
38b2b764 MM |
18 | |
19 | def mv(src, dst): | |
8911f226 | 20 | os.rename(src, dst + '/' + src.split('/')[-1]) |
38b2b764 | 21 | |
54ff0049 ER |
22 | # duplicate code in ftptree.py |
23 | def is_debuginfo(nvr): | |
24 | """ | |
25 | returns true if NVR is debuginfo package and separate debuginfo is enabled | |
26 | """ | |
27 | if not config.separate_debuginfo: | |
28 | return False | |
29 | pkg = nvr.split('-')[:-2] | |
30 | return pkg[-1] == 'debuginfo' | |
31 | ||
38b2b764 MM |
32 | def findfiles(dir): |
33 | def filterinfos(x): | |
8911f226 | 34 | if x[-11:] == '.uploadinfo': |
76dc40c3 | 35 | return True |
38b2b764 MM |
36 | else: |
37 | return False | |
38 | return filter(filterinfos, os.listdir(dir)) | |
39 | ||
38b2b764 | 40 | def getcontent(file): |
8911f226 ER |
41 | f = open(file, 'r') |
42 | content = f.read() | |
38b2b764 | 43 | f.close() |
8911f226 | 44 | if not content[-5:] == '\nEND\n': |
38b2b764 MM |
45 | return None |
46 | else: | |
47 | return content[:-4] | |
48 | ||
8e089d4f | 49 | def send_noarch_msg(files_differ, reqs_differ, pkg, rpmfile, arch): |
200470c5 MM |
50 | req_email=pkg.build[pkg.lastbid].requester_email |
51 | req_bid=pkg.lastbid | |
52 | cc_list=[] | |
0fa16f20 MM |
53 | if 'logs_list' in cval: |
54 | cc_list.append(cval['logs_list']) | |
8e089d4f | 55 | m_subject="Noarch error: %s files differ among builders" % rpmfile |
200470c5 MM |
56 | bids=pkg.build.keys() |
57 | if len(bids)>1: | |
58 | for bid in bids: | |
59 | newcc=pkg.build[bid].requester_email | |
60 | if req_email!=newcc and newcc not in cc_list: | |
61 | cc_list.append(newcc) | |
62 | ||
63 | msg="From: %s\nTo: %s\n" % (cval['from_field'], req_email) | |
64 | if cc_list: | |
65 | msg=msg+"Cc: %s\n" % ", ".join(cc_list) | |
66 | msg=msg+"""X-PLD-Builder: %s | |
67 | References: <%s@pld.src.builder> | |
68 | In-Reply-To: <%s@pld.src.builder> | |
69 | Subject: %s | |
70 | ||
71 | """ % (cval['xpldbuilder'], req_bid, req_bid, m_subject) | |
72 | ||
73 | sm = os.popen("/usr/sbin/sendmail -t", "w") | |
8e089d4f | 74 | |
200470c5 MM |
75 | sm.write(msg) |
76 | ||
77 | if files_differ: | |
78 | f=open("%s/files.diff" % tmpdir, 'r') | |
79 | sm.write("Difference between %s (currently in %s) and %s FILES\n" % | |
8e089d4f | 80 | (pkg.noarch_arch[rpmfile], `ftptree`, arch)), |
200470c5 MM |
81 | for line in f.readlines(True)[2:]: |
82 | sm.write(line) | |
83 | f.close() | |
84 | ||
85 | sm.write('\n') | |
86 | ||
87 | if reqs_differ: | |
88 | f=open("%s/reqs.diff" % tmpdir, 'r') | |
89 | sm.write("Difference between %s (currently in %s) and %s REQS\n" % | |
8e089d4f | 90 | (pkg.noarch_arch[rpmfile], `ftptree`, arch)), |
200470c5 MM |
91 | for line in f.readlines(True)[2:]: |
92 | sm.write(line) | |
93 | f.close() | |
94 | ||
95 | sm.close() | |
96 | ||
8e089d4f ER |
97 | def move_noarch(f, arch, rpmfile, dstpkg): |
98 | if dstpkg.noarch_arch.has_key(rpmfile): | |
a999d554 | 99 | os.system("LC_ALL=C rpm -qlp %s | LC_ALL=C sort > %s/files.new" % |
8e089d4f | 100 | (incoming_dir + arch + '/' + rpmfile, tmpdir)) |
ca2e224a | 101 | os.system("rpm -qRp %s | LC_ALL=C sort | LC_ALL=C uniq > %s/reqs.new" % |
8e089d4f | 102 | (incoming_dir + arch + '/' + rpmfile, tmpdir)) |
bb2cb325 | 103 | |
8911f226 ER |
104 | files_differ = False |
105 | reqs_differ = False | |
bb2cb325 MM |
106 | |
107 | if os.system("diff -u %s/%s.filelist %s/files.new > %s/files.diff" % | |
8e089d4f | 108 | (noarchcachedir, rpmfile, tmpdir, tmpdir)): |
8911f226 | 109 | files_differ = True |
bb2cb325 | 110 | if os.system("diff -u %s/%s.reqlist %s/reqs.new > %s/reqs.diff" % |
8e089d4f | 111 | (noarchcachedir, rpmfile, tmpdir, tmpdir)): |
8911f226 | 112 | reqs_differ = True |
bb2cb325 MM |
113 | |
114 | if files_differ or reqs_differ: | |
8e089d4f | 115 | send_noarch_msg(files_differ, reqs_differ, dstpkg, rpmfile, arch) |
bb2cb325 | 116 | |
8e089d4f | 117 | rm(incoming_dir + arch + '/' + rpmfile) |
bb2cb325 | 118 | else: |
ca2e224a | 119 | os.system("LC_ALL=C rpm -qlp %s | LC_ALL=C sort > %s/%s.filelist" % |
8e089d4f | 120 | (incoming_dir + arch + '/' + rpmfile, noarchcachedir, rpmfile)) |
ca2e224a | 121 | os.system("rpm -qRp %s | LC_ALL=C sort | LC_ALL=C uniq > %s/%s.reqlist" % |
8e089d4f | 122 | (incoming_dir + arch + '/' + rpmfile, noarchcachedir, rpmfile)) |
c0d14f9d JR |
123 | if not dstpkg.files.has_key(arch): |
124 | f.write("file:noarch:%s\ninfo:noarch_arch:%s:%s\n" % (rpmfile, rpmfile, arch)) | |
8e089d4f | 125 | mv(incoming_dir + arch + '/' + rpmfile, default_to + 'noarch/RPMS') |
38b2b764 | 126 | |
c77db9e7 JR |
127 | def send_vr_msg(snvr, anvr, pkg, arch): |
128 | req_email=pkg.build[pkg.lastbid].requester_email | |
129 | req_bid=pkg.lastbid | |
130 | cc_list=[] | |
131 | if 'logs_list' in cval: | |
132 | cc_list.append(cval['logs_list']) | |
133 | m_subject="NVR error: %s version or relese differ among subpackages" % snvr[0] | |
134 | bids=pkg.build.keys() | |
135 | if len(bids)>1: | |
136 | for bid in bids: | |
137 | newcc=pkg.build[bid].requester_email | |
138 | if req_email!=newcc and newcc not in cc_list: | |
139 | cc_list.append(newcc) | |
140 | ||
141 | msg="From: %s\nTo: %s\n" % (cval['from_field'], req_email) | |
142 | if cc_list: | |
143 | msg=msg+"Cc: %s\n" % ", ".join(cc_list) | |
144 | msg=msg+"""X-PLD-Builder: %s | |
145 | References: <%s@pld.src.builder> | |
146 | In-Reply-To: <%s@pld.src.builder> | |
147 | Subject: %s | |
148 | ||
149 | """ % (cval['xpldbuilder'], req_bid, req_bid, m_subject) | |
150 | ||
151 | sm = os.popen("/usr/sbin/sendmail -t", "w") | |
152 | ||
153 | sm.write(msg) | |
154 | ||
155 | sm.write("Difference between %s SRPM (currently in %s) and %s RPM NVR:\n\n" % | |
156 | (snvr[0], `ftptree`, arch)), | |
c82d0ed4 JR |
157 | sm.write("Expected (%s):\nV: %s\nR: %s\n\n" % snvr) |
158 | sm.write("RPM:\nN: %s\nV: %s\nR: %s\n" % anvr) | |
c77db9e7 JR |
159 | sm.write('\n') |
160 | ||
161 | sm.close() | |
162 | ||
e4b21e2b ER |
163 | # main() |
164 | try: | |
165 | ftpio.connect('from-incoming-pid-%s' % os.getpid()) | |
166 | except: | |
65017da5 ER |
167 | print "Can't get ftpiod connection" |
168 | sys.exit(1) | |
e4b21e2b ER |
169 | |
170 | ftptree = BaseFtpTree(cval['default_to']) | |
171 | ||
172 | if not ftpio.lock(cval['default_to']): | |
65017da5 ER |
173 | print "Can't get lock: %s" % cval['default_to'] |
174 | sys.exit(1) | |
e4b21e2b | 175 | |
7c2f5e4c AM |
176 | moved_anything = False |
177 | ||
e4b21e2b ER |
178 | for uploadinfo in findfiles(incoming_dir + 'SRPMS'): |
179 | content = getcontent(incoming_dir + 'SRPMS/' + uploadinfo) | |
180 | if not content: | |
181 | continue # Uploading not finished | |
182 | ||
183 | pkg = BasePkg(uploadinfo[:-19], content = content) | |
184 | srpm = pkg.files['SRPMS'][0] | |
185 | ||
46d96a92 AM |
186 | if not os.path.exists(incoming_dir + 'SRPMS/' + srpm): |
187 | ftpio.log("%s file missing; skipping move until next round" % (srpm)) | |
188 | continue | |
189 | ||
e4b21e2b | 190 | if ftptree.has_key(`pkg`): |
99b125ec JR |
191 | ftpio.log("%s already present in %s; removing older files" % (srpm, ftptree)) |
192 | rm(default_to + 'SRPMS/RPMS/' + srpm) | |
e4b21e2b ER |
193 | f = open(default_to + 'SRPMS/.metadata/' + srpm+'.info', 'a') |
194 | bid = pkg.build.keys()[0] | |
195 | build = pkg.build[bid] | |
196 | f.write("info:build:%s:requester:%s\ninfo:build:%s:requester_email:%s\n" | |
197 | % (bid, build.requester, bid, build.requester_email)) | |
198 | f.close() | |
199 | else: | |
e4b21e2b ER |
200 | f = open(default_to + 'SRPMS/.metadata/' + srpm + '.info', 'w') |
201 | f.write(content) | |
202 | f.close() | |
203 | ||
99b125ec | 204 | mv(incoming_dir + 'SRPMS/' + srpm, default_to + 'SRPMS/RPMS') |
e4b21e2b ER |
205 | rm(incoming_dir + 'SRPMS/' + uploadinfo) |
206 | ||
38b2b764 | 207 | for arch in ftp_archs: |
8e089d4f | 208 | for uploadinfo in findfiles(incoming_dir + arch): |
8911f226 | 209 | content = getcontent(incoming_dir + arch + '/' + uploadinfo) |
38b2b764 | 210 | if not content: |
2abd3b77 | 211 | ftpio.log("%s not finished uploading" % uploadinfo) |
38b2b764 MM |
212 | continue # Uploading not finished |
213 | ||
8911f226 ER |
214 | srcpkg = BasePkg(uploadinfo[:-19], content = content) |
215 | srpm = srcpkg.files['SRPMS'][0] | |
38b2b764 | 216 | |
bb2cb325 | 217 | if not ftptree.has_key(`srcpkg`): |
38b2b764 MM |
218 | continue # We require the src.rpm to be present |
219 | ||
c77db9e7 JR |
220 | renvr = re.compile(r'(.*)-(.*)-(.*)\.[^.]*\.rpm') |
221 | srcnvr = renvr.match(srpm).groups() | |
222 | ||
46d96a92 AM |
223 | rpmfile_missing = [f for f in srcpkg.files['ARCH'] if not os.path.exists(incoming_dir + arch + '/'+f)] |
224 | if len(rpmfile_missing): | |
225 | for filem in rpmfile_missing: | |
226 | ftpio.log("%s file missing; skipping move until next round" % (filem)) | |
227 | continue | |
228 | ||
8911f226 | 229 | dstpkg = BasePkg(`srcpkg`, ftptree) |
38b2b764 | 230 | |
bb2cb325 | 231 | if dstpkg.files.has_key(arch): |
99b125ec JR |
232 | ftpio.log("files from %s for arch %s already present in %s; removing older files" % (`srcpkg`, arch, ftptree)) |
233 | for rpmfile in dstpkg.files[arch]: | |
234 | if is_debuginfo(rpmfile): | |
235 | dstfile = default_to + arch + '/debuginfo' | |
236 | else: | |
237 | dstfile = default_to + arch + '/RPMS' | |
2eb653d6 | 238 | try: |
99b125ec | 239 | rm(dstfile + '/' + rpmfile) |
2eb653d6 | 240 | except OSError, e: |
99b125ec | 241 | l = "Removing %s problem: %s" % (dstfile + '/' + rpmfile, e) |
2eb653d6 AM |
242 | ftpio.log(l) |
243 | print l | |
38b2b764 | 244 | |
8911f226 | 245 | f = open(default_to + 'SRPMS/.metadata/' + srpm + '.info', 'a') |
8e089d4f | 246 | for rpmfile in srcpkg.files['ARCH']: |
7c2f5e4c | 247 | moved_anything = True |
c77db9e7 | 248 | |
164d8d15 JR |
249 | # Too much noise, too little use |
250 | # archnvr = renvr.match(rpmfile).groups() | |
251 | # if srcnvr[1] != archnvr[1] or srcnvr[2] != archnvr[2]: | |
252 | # send_vr_msg(srcnvr, archnvr, dstpkg, arch) | |
c77db9e7 | 253 | |
8e089d4f ER |
254 | if rpmfile[-11:] == '.noarch.rpm' and config.separate_noarch: |
255 | move_noarch(f, arch, rpmfile, dstpkg) | |
bb2cb325 | 256 | else: |
c0d14f9d JR |
257 | if not dstpkg.files.has_key(arch): |
258 | f.write("file:%s:%s\n" % (arch, rpmfile)) | |
8e089d4f | 259 | srcfile = incoming_dir + arch + '/' + rpmfile |
8911f226 | 260 | |
8e089d4f | 261 | if is_debuginfo(rpmfile): |
54ff0049 ER |
262 | dstfile = default_to + arch + '/debuginfo' |
263 | else: | |
264 | dstfile = default_to + arch + '/RPMS' | |
8911f226 | 265 | |
631d11b9 | 266 | try: |
8911f226 | 267 | mv(srcfile, dstfile) |
631d11b9 | 268 | except OSError, e: |
8911f226 | 269 | l = "Moving %s to %s problem: %s" % (srcfile, dstfile, e) |
631d11b9 AM |
270 | ftpio.log(l) |
271 | print l | |
38b2b764 MM |
272 | f.close() |
273 | ||
8911f226 | 274 | rm(incoming_dir + arch + '/' + uploadinfo) |
38b2b764 | 275 | |
0a108b7f | 276 | ftpio.unlock(cval['default_to']) |
7c2f5e4c AM |
277 | |
278 | if moved_anything: | |
a34f0e55 | 279 | os.system("%s/pld-ftp-admin/bin/pfa-genindex --quiet test > /dev/null" % (os.getenv("HOME"))) |