]> git.pld-linux.org Git - projects/pld-builder.new.git/blame - PLD_Builder/rpm_builder.py
New way of skipping requests.
[projects/pld-builder.new.git] / PLD_Builder / rpm_builder.py
CommitLineData
dfff8bd5
MM
1# vi: encoding=utf-8 ts=8 sts=4 sw=4 et
2
3ea4b156
MM
3import sys
4import os
5import atexit
6import time
deda9a51 7import string
3ea4b156
MM
8import urllib
9
10from config import config, init_conf
11from bqueue import B_Queue
3ea4b156
MM
12import lock
13import util
3f446d8f 14import loop
3ea4b156
MM
15import path
16import status
17import log
18import chroot
19import ftp
e2cad913 20import buildlogs
59ce7cd6 21import notify
17f23d66
MM
22import build
23import report
73bdb36b 24import install
3ea4b156 25
53989cf3
MM
26# *HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*
27import socket
28
29socket.myorigsocket=socket.socket
30
31def mysocket(family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0):
dfff8bd5
MM
32 s=socket.myorigsocket(family, type, proto)
33 s.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
34 return s
53989cf3
MM
35
36socket.socket=mysocket
37# *HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*
38
3ea4b156
MM
39# this code is duplicated in srpm_builder, but we
40# might want to handle some cases differently here
41def pick_request(q):
dfff8bd5
MM
42 def mycmp(r1, r2):
43 if r1.kind != 'group' or r2.kind != 'group':
939af6d7 44 raise Exception, "non-group requests"
dfff8bd5
MM
45 pri_diff = cmp(r1.priority, r2.priority)
46 if pri_diff == 0:
47 return cmp(r1.time, r2.time)
48 else:
bbed4c4d 49 return pri_diff
dfff8bd5
MM
50 q.requests.sort(mycmp)
51 ret = q.requests[0]
52 return ret
3ea4b156 53
6ce49c66
AM
54def check_skip_build(r):
55 src_url = config.control_url + "/srpms/" + r.id + "/skipme"
56 good = True
57 while not good:
58 try:
59 good=True
60 f = urllib.urlopen(src_url)
61 except IOError, error:
62 if error[1][0] == 60 or error[1][0] == 110 or error[1][0] == -3 or error[1][0] == 111 or error[1][0] == 61:
63 good=False
64 b.log_line("unable to connect... trying again")
65 continue
66 else:
67 return False
68 http_code = f.getcode()
69 if http_code == 200:
70 f.close()
71 return True
72 return False
73
17f23d66 74def fetch_src(r, b):
dfff8bd5
MM
75 src_url = config.control_url + "/srpms/" + r.id + "/" + b.src_rpm
76 b.log_line("fetching %s" % src_url)
77 start = time.time()
78 good=False
79 while not good:
80 try:
81 good=True
82 f = urllib.urlopen(src_url)
83 except IOError, error:
84 if error[1][0] == 60 or error[1][0] == 110 or error[1][0] == -3 or error[1][0] == 111 or error[1][0] == 61:
85 good=False
86 b.log_line("unable to connect... trying again")
e0226a74 87 continue
dfff8bd5 88 else:
85d03545 89 raise
51157699
AM
90 http_code = f.getcode()
91 if http_code != 200:
6ce49c66
AM
92 # fail in a way where cron job will retry
93 msg = "unable to fetch file, http code: %d" % http_code
94 b.log_line(msg)
9bda2ae6 95 f.close()
6ce49c66 96 raise IOError, msg
dfff8bd5
MM
97
98 o = chroot.popen("cat > %s" % b.src_rpm, mode = "w")
9e89ce60
AM
99
100 try:
101 bytes = util.sendfile(f, o)
102 except IOError, e:
00a1db68 103 b.log_line("error: unable to write to `%s': %s" % (b.src_rpm, e))
9e89ce60
AM
104 raise
105
dfff8bd5
MM
106 f.close()
107 o.close()
108 t = time.time() - start
109 if t == 0:
110 b.log_line("fetched %d bytes" % bytes)
111 else:
112 b.log_line("fetched %d bytes, %.1f K/s" % (bytes, bytes / 1024.0 / t))
3ea4b156 113
84c6175b
MM
114def prepare_env():
115 chroot.run("test ! -f /proc/uptime && mount /proc", 'root')
9b2ab0b4
AM
116 chroot.run("test ! -e /dev/full && mknod /dev/full c 1 7 && chmod 666 /dev/full", 'root')
117 chroot.run("test ! -e /dev/null && mknod /dev/null c 1 3 && chmod 666 /dev/null", 'root')
118 chroot.run("test ! -e /dev/random && mknod /dev/random c 1 8 && chmod 644 /dev/random", 'root')
119 chroot.run("test ! -e /dev/urandom && mknod /dev/urandom c 1 9 && chmod 644 /dev/urandom", 'root')
120 chroot.run("test ! -e /dev/zero && mknod /dev/zero c 1 5 && chmod 666 /dev/zero", 'root')
62aff600 121 # try to limit network access for builder account
52a64a7d 122 chroot.run("/bin/setfacl -m u:builder:--- /etc/resolv.conf", 'root')
84c6175b 123
17f23d66 124def build_rpm(r, b):
dfff8bd5
MM
125 status.push("building %s" % b.spec)
126 b.log_line("request from: %s" % r.requester)
6ce49c66
AM
127
128 if check_skip_build(r):
129 b.log_line("build skipped due to src builder request")
130 res = "SKIP_REQUESTED"
131 return res
132
dfff8bd5
MM
133 b.log_line("started at: %s" % time.asctime())
134 fetch_src(r, b)
135 b.log_line("installing srpm: %s" % b.src_rpm)
136 res = chroot.run("rpm -U %s" % b.src_rpm, logfile = b.logfile)
137 chroot.run("rm -f %s" % b.src_rpm, logfile = b.logfile)
138 b.files = []
139 tmpdir = "/tmp/B." + b.b_id[0:6]
140 if res:
141 b.log_line("error: installing src rpm failed")
511ece79 142 res = "FAIL_SRPM_INSTALL"
deda9a51 143 else:
84c6175b 144 prepare_env()
dfff8bd5 145 chroot.run("install -m 700 -d %s" % tmpdir)
be264f26
ER
146
147 b.default_target(config.arch)
2e5736d2 148 rpmbuild_opt = "%s %s %s" % (b.target_string(), b.kernel_string(), b.bconds_string())
81c135f6
ER
149 # check for build arch before filling BR
150 cmd = "cd rpm/SPECS; TMPDIR=%s nice -n %s rpmbuild -bp --short-circuit --nodeps --define 'prep exit 0' %s %s" % \
151 (tmpdir, config.nice, rpmbuild_opt, b.spec)
152 res = chroot.run(cmd, logfile = b.logfile)
4f2d26fb 153 if res:
511ece79 154 res = "UNSUPP"
a957e64f 155 b.log_line("error: build arch check (%s) failed" % cmd)
81c135f6
ER
156
157 if not res:
feb26777 158 if ("no-install-br" not in r.flags) and not install.uninstall_self_conflict(b):
511ece79 159 res = "FAIL_DEPS_UNINSTALL"
feb26777 160 if ("no-install-br" not in r.flags) and not install.install_br(r, b):
511ece79 161 res = "FAIL_DEPS_INSTALL"
73bdb36b 162 if not res:
a1b28a50 163 max_jobs = max(min(int(os.sysconf('SC_NPROCESSORS_ONLN') * 1.5), config.max_jobs), 1)
85303822 164 if r.max_jobs > 0:
a1b28a50 165 max_jobs = max(min(config.max_jobs, r.max_jobs), 1)
b3c8d962 166 cmd = "cd rpm/SPECS; TMPDIR=%s nice -n %s rpmbuild -bb --define '_smp_mflags -j%d' %s %s" % \
85303822 167 (tmpdir, config.nice, max_jobs, rpmbuild_opt, b.spec)
81c135f6 168 b.log_line("building RPM using: %s" % cmd)
efb3621d 169 begin_time = time.time()
81c135f6 170 res = chroot.run(cmd, logfile = b.logfile)
efb3621d 171 end_time = time.time()
200299dd 172 b.log_line("ended at: %s, done in %s" % (time.asctime(), time.strftime("%Hh %Mm %Ss", time.gmtime(end_time - begin_time))))
4f2d26fb
AM
173 if res:
174 res = "FAIL"
81c135f6
ER
175 files = util.collect_files(b.logfile)
176 if len(files) > 0:
177 r.chroot_files.extend(files)
178 else:
179 b.log_line("error: No files produced.")
2b9e7381
AM
180 last_section = util.find_last_section(b.logfile)
181 if last_section == None:
182 res = "FAIL"
183 else:
184 res = "FAIL_%s" % last_section.upper()
81c135f6
ER
185 b.files = files
186
dfff8bd5
MM
187 chroot.run("rm -rf %s; cd rpm/SPECS; rpmbuild --nodeps --nobuild " \
188 "--clean --rmspec --rmsource %s" % \
189 (tmpdir, b.spec), logfile = b.logfile)
190 chroot.run("rm -rf $HOME/rpm/BUILD/*")
191
192 def ll(l):
193 util.append_to(b.logfile, l)
deda9a51 194
dfff8bd5 195 if b.files != []:
fd499dc1 196 rpm_cache_dir = config.rpm_cache_dir
e936beda 197 if "test-build" not in r.flags:
adc1235e 198 # NOTE: copying to cache dir doesn't mean that build failed, so ignore result
90d9480e 199 b.log_line("copy rpm files to cache_dir: %s" % rpm_cache_dir)
adc1235e 200 chroot.run(
fd499dc1
ER
201 "cp -f %s %s && poldek --mo=nodiff --mkidxz -s %s/" % \
202 (string.join(b.files), rpm_cache_dir, rpm_cache_dir),
203 logfile = b.logfile, user = "root"
204 )
e936beda 205 else:
fd499dc1 206 ll("test-build: not copying to " + rpm_cache_dir)
dfff8bd5
MM
207 ll("Begin-PLD-Builder-Info")
208 if "upgrade" in r.flags:
73bdb36b 209 b.upgraded = install.upgrade_from_batch(r, b)
dfff8bd5
MM
210 else:
211 ll("not upgrading")
212 ll("End-PLD-Builder-Info")
deda9a51 213
1051562e 214 for f in b.files:
dfff8bd5 215 local = r.tmp_dir + os.path.basename(f)
3e689257 216 chroot.cp(f, outfile = local, rm = True)
dfff8bd5 217 ftp.add(local)
1051562e 218
dfff8bd5
MM
219 def uploadinfo(b):
220 c="file:SRPMS:%s\n" % b.src_rpm
221 for f in b.files:
222 c=c + "file:ARCH:%s\n" % os.path.basename(f)
223 c=c + "END\n"
224 return c
3ea4b156 225
71a06b82 226 if config.gen_upinfo and b.files != [] and 'test-build' not in r.flags:
dfff8bd5
MM
227 fname = r.tmp_dir + b.src_rpm + ".uploadinfo"
228 f = open(fname, "w")
229 f.write(uploadinfo(b))
230 f.close()
231 ftp.add(fname, "uploadinfo")
232
233 status.pop()
234
235 return res
3ea4b156 236
17f23d66 237def handle_request(r):
dfff8bd5
MM
238 ftp.init(r)
239 buildlogs.init(r)
240 build.build_all(r, build_rpm)
241 report.send_report(r, is_src = False)
242 ftp.flush()
63319e0a 243 notify.send(r)
3ea4b156
MM
244
245def check_load():
dfff8bd5
MM
246 do_exit = 0
247 try:
248 f = open("/proc/loadavg")
249 if float(string.split(f.readline())[2]) > config.max_load:
250 do_exit = 1
251 except:
252 pass
253 if do_exit:
254 sys.exit(0)
3ea4b156 255
e8ee9db8 256def main_for(builder):
b8ec18af
AM
257 msg = ""
258
dfff8bd5 259 init_conf(builder)
dfff8bd5 260
dfff8bd5
MM
261 q = B_Queue(path.queue_file + "-" + config.builder)
262 q.lock(0)
263 q.read()
264 if q.requests == []:
265 q.unlock()
266 return
267 req = pick_request(q)
7b16924c 268 q.unlock()
dfff8bd5 269
91112237
AM
270 # high priority tasks have priority < 0, normal tasks >= 0
271 if req.priority >= 0:
b8ec18af
AM
272
273 # allow only one build in given builder at once
274 if not lock.lock("building-rpm-for-%s" % config.builder, non_block = 1):
275 return
276 # don't kill server
277 check_load()
278 # not more then job_slots builds at once
279 locked = 0
280 for slot in range(config.job_slots):
281 if lock.lock("building-rpm-slot-%d" % slot, non_block = 1):
282 locked = 1
283 break
284 if not locked:
285 return
286
287 # record fact that we got lock for this builder, load balancer
288 # will use it for fair-queuing
289 l = lock.lock("got-lock")
290 f = open(path.got_lock_file, "a")
291 f.write(config.builder + "\n")
292 f.close()
293 l.close()
294 else:
295 msg = "HIGH PRIORITY: "
dfff8bd5 296
b8ec18af
AM
297 msg += "handling request %s (%d) for %s from %s, priority %s" \
298 % (req.id, req.no, config.builder, req.requester, req.priority)
dfff8bd5
MM
299 log.notice(msg)
300 status.push(msg)
301 handle_request(req)
302 status.pop()
303
304 def otherreqs(r):
305 if r.no==req.no:
306 return False
307 else:
308 return True
309
310 q = B_Queue(path.queue_file + "-" + config.builder)
311 q.lock(0)
312 q.read()
313 previouslen=len(q.requests)
314 q.requests=filter(otherreqs, q.requests)
315 if len(q.requests)<previouslen:
316 q.write()
317 q.unlock()
318
e8ee9db8 319def main():
dfff8bd5 320 if len(sys.argv) < 2:
939af6d7 321 raise Exception, "fatal: need to have builder name as first arg"
dfff8bd5
MM
322 return main_for(sys.argv[1])
323
e8ee9db8 324if __name__ == '__main__':
dfff8bd5 325 loop.run_loop(main)
This page took 0.09237 seconds and 4 git commands to generate.