]> git.pld-linux.org Git - projects/pld-ftp-admin.git/blame - bin/pfa-from-incoming
Always overwrite files in dest tree with newly built ones
[projects/pld-ftp-admin.git] / bin / pfa-from-incoming
CommitLineData
38b2b764
MM
1#!/usr/bin/env python
2# vi: encoding=utf-8 ts=8 sts=4 sw=4 et
3
c77db9e7 4import sys, os, stat, time, re
38b2b764
MM
5sys.path.insert(0, os.environ['HOME']+'/pld-ftp-admin/modules')
6from config import incoming_dir, default_to, ftp_archs
8e089d4f 7from config import value as cval
38b2b764 8import config
acd976ec 9from common import noarchcachedir, tmpdir, fileexists
bb2cb325 10from baseftptree import BaseFtpTree, BasePkg
8e089d4f 11from ftptree import FtpTree, Pkg
0a108b7f 12import ftpio
38b2b764 13
fc574b00
AM
14os.umask(022)
15
38b2b764 16def rm(file):
18c8e935 17 os.remove(file)
38b2b764
MM
18
19def mv(src, dst):
8911f226 20 os.rename(src, dst + '/' + src.split('/')[-1])
38b2b764 21
54ff0049
ER
22# duplicate code in ftptree.py
23def is_debuginfo(nvr):
24 """
25 returns true if NVR is debuginfo package and separate debuginfo is enabled
26 """
27 if not config.separate_debuginfo:
28 return False
29 pkg = nvr.split('-')[:-2]
30 return pkg[-1] == 'debuginfo'
31
38b2b764
MM
32def findfiles(dir):
33 def filterinfos(x):
8911f226 34 if x[-11:] == '.uploadinfo':
76dc40c3 35 return True
38b2b764
MM
36 else:
37 return False
38 return filter(filterinfos, os.listdir(dir))
39
38b2b764 40def getcontent(file):
8911f226
ER
41 f = open(file, 'r')
42 content = f.read()
38b2b764 43 f.close()
8911f226 44 if not content[-5:] == '\nEND\n':
38b2b764
MM
45 return None
46 else:
47 return content[:-4]
48
8e089d4f 49def send_noarch_msg(files_differ, reqs_differ, pkg, rpmfile, arch):
200470c5
MM
50 req_email=pkg.build[pkg.lastbid].requester_email
51 req_bid=pkg.lastbid
52 cc_list=[]
0fa16f20
MM
53 if 'logs_list' in cval:
54 cc_list.append(cval['logs_list'])
8e089d4f 55 m_subject="Noarch error: %s files differ among builders" % rpmfile
200470c5
MM
56 bids=pkg.build.keys()
57 if len(bids)>1:
58 for bid in bids:
59 newcc=pkg.build[bid].requester_email
60 if req_email!=newcc and newcc not in cc_list:
61 cc_list.append(newcc)
62
63 msg="From: %s\nTo: %s\n" % (cval['from_field'], req_email)
64 if cc_list:
65 msg=msg+"Cc: %s\n" % ", ".join(cc_list)
66 msg=msg+"""X-PLD-Builder: %s
67References: <%s@pld.src.builder>
68In-Reply-To: <%s@pld.src.builder>
69Subject: %s
70
71""" % (cval['xpldbuilder'], req_bid, req_bid, m_subject)
72
73 sm = os.popen("/usr/sbin/sendmail -t", "w")
8e089d4f 74
200470c5
MM
75 sm.write(msg)
76
77 if files_differ:
78 f=open("%s/files.diff" % tmpdir, 'r')
79 sm.write("Difference between %s (currently in %s) and %s FILES\n" %
8e089d4f 80 (pkg.noarch_arch[rpmfile], `ftptree`, arch)),
200470c5
MM
81 for line in f.readlines(True)[2:]:
82 sm.write(line)
83 f.close()
84
85 sm.write('\n')
86
87 if reqs_differ:
88 f=open("%s/reqs.diff" % tmpdir, 'r')
89 sm.write("Difference between %s (currently in %s) and %s REQS\n" %
8e089d4f 90 (pkg.noarch_arch[rpmfile], `ftptree`, arch)),
200470c5
MM
91 for line in f.readlines(True)[2:]:
92 sm.write(line)
93 f.close()
94
95 sm.close()
96
8e089d4f
ER
97def move_noarch(f, arch, rpmfile, dstpkg):
98 if dstpkg.noarch_arch.has_key(rpmfile):
a999d554 99 os.system("LC_ALL=C rpm -qlp %s | LC_ALL=C sort > %s/files.new" %
8e089d4f 100 (incoming_dir + arch + '/' + rpmfile, tmpdir))
ca2e224a 101 os.system("rpm -qRp %s | LC_ALL=C sort | LC_ALL=C uniq > %s/reqs.new" %
8e089d4f 102 (incoming_dir + arch + '/' + rpmfile, tmpdir))
bb2cb325 103
8911f226
ER
104 files_differ = False
105 reqs_differ = False
bb2cb325
MM
106
107 if os.system("diff -u %s/%s.filelist %s/files.new > %s/files.diff" %
8e089d4f 108 (noarchcachedir, rpmfile, tmpdir, tmpdir)):
8911f226 109 files_differ = True
bb2cb325 110 if os.system("diff -u %s/%s.reqlist %s/reqs.new > %s/reqs.diff" %
8e089d4f 111 (noarchcachedir, rpmfile, tmpdir, tmpdir)):
8911f226 112 reqs_differ = True
bb2cb325
MM
113
114 if files_differ or reqs_differ:
8e089d4f 115 send_noarch_msg(files_differ, reqs_differ, dstpkg, rpmfile, arch)
bb2cb325 116
8e089d4f 117 rm(incoming_dir + arch + '/' + rpmfile)
bb2cb325 118 else:
ca2e224a 119 os.system("LC_ALL=C rpm -qlp %s | LC_ALL=C sort > %s/%s.filelist" %
8e089d4f 120 (incoming_dir + arch + '/' + rpmfile, noarchcachedir, rpmfile))
ca2e224a 121 os.system("rpm -qRp %s | LC_ALL=C sort | LC_ALL=C uniq > %s/%s.reqlist" %
8e089d4f
ER
122 (incoming_dir + arch + '/' + rpmfile, noarchcachedir, rpmfile))
123 f.write("file:noarch:%s\ninfo:noarch_arch:%s:%s\n" % (rpmfile, rpmfile, arch))
124 mv(incoming_dir + arch + '/' + rpmfile, default_to + 'noarch/RPMS')
38b2b764 125
c77db9e7
JR
126def send_vr_msg(snvr, anvr, pkg, arch):
127 req_email=pkg.build[pkg.lastbid].requester_email
128 req_bid=pkg.lastbid
129 cc_list=[]
130 if 'logs_list' in cval:
131 cc_list.append(cval['logs_list'])
132 m_subject="NVR error: %s version or relese differ among subpackages" % snvr[0]
133 bids=pkg.build.keys()
134 if len(bids)>1:
135 for bid in bids:
136 newcc=pkg.build[bid].requester_email
137 if req_email!=newcc and newcc not in cc_list:
138 cc_list.append(newcc)
139
140 msg="From: %s\nTo: %s\n" % (cval['from_field'], req_email)
141 if cc_list:
142 msg=msg+"Cc: %s\n" % ", ".join(cc_list)
143 msg=msg+"""X-PLD-Builder: %s
144References: <%s@pld.src.builder>
145In-Reply-To: <%s@pld.src.builder>
146Subject: %s
147
148""" % (cval['xpldbuilder'], req_bid, req_bid, m_subject)
149
150 sm = os.popen("/usr/sbin/sendmail -t", "w")
151
152 sm.write(msg)
153
154 sm.write("Difference between %s SRPM (currently in %s) and %s RPM NVR:\n\n" %
155 (snvr[0], `ftptree`, arch)),
c82d0ed4
JR
156 sm.write("Expected (%s):\nV: %s\nR: %s\n\n" % snvr)
157 sm.write("RPM:\nN: %s\nV: %s\nR: %s\n" % anvr)
c77db9e7
JR
158 sm.write('\n')
159
160 sm.close()
161
e4b21e2b
ER
162# main()
163try:
164 ftpio.connect('from-incoming-pid-%s' % os.getpid())
165except:
65017da5
ER
166 print "Can't get ftpiod connection"
167 sys.exit(1)
e4b21e2b
ER
168
169ftptree = BaseFtpTree(cval['default_to'])
170
171if not ftpio.lock(cval['default_to']):
65017da5
ER
172 print "Can't get lock: %s" % cval['default_to']
173 sys.exit(1)
e4b21e2b 174
7c2f5e4c
AM
175moved_anything = False
176
e4b21e2b
ER
177for uploadinfo in findfiles(incoming_dir + 'SRPMS'):
178 content = getcontent(incoming_dir + 'SRPMS/' + uploadinfo)
179 if not content:
180 continue # Uploading not finished
181
182 pkg = BasePkg(uploadinfo[:-19], content = content)
183 srpm = pkg.files['SRPMS'][0]
184
46d96a92
AM
185 if not os.path.exists(incoming_dir + 'SRPMS/' + srpm):
186 ftpio.log("%s file missing; skipping move until next round" % (srpm))
187 continue
188
e4b21e2b 189 if ftptree.has_key(`pkg`):
99b125ec
JR
190 ftpio.log("%s already present in %s; removing older files" % (srpm, ftptree))
191 rm(default_to + 'SRPMS/RPMS/' + srpm)
e4b21e2b
ER
192 f = open(default_to + 'SRPMS/.metadata/' + srpm+'.info', 'a')
193 bid = pkg.build.keys()[0]
194 build = pkg.build[bid]
195 f.write("info:build:%s:requester:%s\ninfo:build:%s:requester_email:%s\n"
196 % (bid, build.requester, bid, build.requester_email))
197 f.close()
198 else:
e4b21e2b
ER
199 f = open(default_to + 'SRPMS/.metadata/' + srpm + '.info', 'w')
200 f.write(content)
201 f.close()
202
99b125ec 203 mv(incoming_dir + 'SRPMS/' + srpm, default_to + 'SRPMS/RPMS')
e4b21e2b
ER
204 rm(incoming_dir + 'SRPMS/' + uploadinfo)
205
38b2b764 206for arch in ftp_archs:
8e089d4f 207 for uploadinfo in findfiles(incoming_dir + arch):
8911f226 208 content = getcontent(incoming_dir + arch + '/' + uploadinfo)
38b2b764 209 if not content:
2abd3b77 210 ftpio.log("%s not finished uploading" % uploadinfo)
38b2b764
MM
211 continue # Uploading not finished
212
8911f226
ER
213 srcpkg = BasePkg(uploadinfo[:-19], content = content)
214 srpm = srcpkg.files['SRPMS'][0]
38b2b764 215
bb2cb325 216 if not ftptree.has_key(`srcpkg`):
38b2b764
MM
217 continue # We require the src.rpm to be present
218
c77db9e7
JR
219 renvr = re.compile(r'(.*)-(.*)-(.*)\.[^.]*\.rpm')
220 srcnvr = renvr.match(srpm).groups()
221
46d96a92
AM
222 rpmfile_missing = [f for f in srcpkg.files['ARCH'] if not os.path.exists(incoming_dir + arch + '/'+f)]
223 if len(rpmfile_missing):
224 for filem in rpmfile_missing:
225 ftpio.log("%s file missing; skipping move until next round" % (filem))
226 continue
227
8911f226 228 dstpkg = BasePkg(`srcpkg`, ftptree)
38b2b764 229
bb2cb325 230 if dstpkg.files.has_key(arch):
99b125ec
JR
231 ftpio.log("files from %s for arch %s already present in %s; removing older files" % (`srcpkg`, arch, ftptree))
232 for rpmfile in dstpkg.files[arch]:
233 if is_debuginfo(rpmfile):
234 dstfile = default_to + arch + '/debuginfo'
235 else:
236 dstfile = default_to + arch + '/RPMS'
2eb653d6 237 try:
99b125ec 238 rm(dstfile + '/' + rpmfile)
2eb653d6 239 except OSError, e:
99b125ec 240 l = "Removing %s problem: %s" % (dstfile + '/' + rpmfile, e)
2eb653d6
AM
241 ftpio.log(l)
242 print l
38b2b764 243
8911f226 244 f = open(default_to + 'SRPMS/.metadata/' + srpm + '.info', 'a')
8e089d4f 245 for rpmfile in srcpkg.files['ARCH']:
7c2f5e4c 246 moved_anything = True
c77db9e7 247
164d8d15
JR
248# Too much noise, too little use
249# archnvr = renvr.match(rpmfile).groups()
250# if srcnvr[1] != archnvr[1] or srcnvr[2] != archnvr[2]:
251# send_vr_msg(srcnvr, archnvr, dstpkg, arch)
c77db9e7 252
8e089d4f
ER
253 if rpmfile[-11:] == '.noarch.rpm' and config.separate_noarch:
254 move_noarch(f, arch, rpmfile, dstpkg)
bb2cb325 255 else:
8e089d4f
ER
256 f.write("file:%s:%s\n" % (arch, rpmfile))
257 srcfile = incoming_dir + arch + '/' + rpmfile
8911f226 258
8e089d4f 259 if is_debuginfo(rpmfile):
54ff0049
ER
260 dstfile = default_to + arch + '/debuginfo'
261 else:
262 dstfile = default_to + arch + '/RPMS'
8911f226 263
631d11b9 264 try:
8911f226 265 mv(srcfile, dstfile)
631d11b9 266 except OSError, e:
8911f226 267 l = "Moving %s to %s problem: %s" % (srcfile, dstfile, e)
631d11b9
AM
268 ftpio.log(l)
269 print l
38b2b764
MM
270 f.close()
271
8911f226 272 rm(incoming_dir + arch + '/' + uploadinfo)
38b2b764 273
0a108b7f 274ftpio.unlock(cval['default_to'])
7c2f5e4c
AM
275
276if moved_anything:
a34f0e55 277 os.system("%s/pld-ftp-admin/bin/pfa-genindex --quiet test > /dev/null" % (os.getenv("HOME")))
This page took 0.272828 seconds and 4 git commands to generate.