1 # vi: encoding=utf-8 ts=8 sts=4 sw=4 et
10 from config import config, init_conf
11 from bqueue import B_Queue
26 # *HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*
29 socket.myorigsocket=socket.socket
31 def mysocket(family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0):
32 s=socket.myorigsocket(family, type, proto)
33 s.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
36 socket.socket=mysocket
37 # *HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*
39 # this code is duplicated in srpm_builder, but we
40 # might want to handle some cases differently here
43 if r1.kind != 'group' or r2.kind != 'group':
44 raise Exception, "non-group requests"
45 pri_diff = cmp(r1.priority, r2.priority)
47 return cmp(r1.time, r2.time)
50 q.requests.sort(mycmp)
54 def check_skip_build(r, b):
55 src_url = config.control_url + "/srpms/" + r.id + "/skipme"
57 b.log_line("checking if we should skip the build")
60 headers = { 'Cache-Control': 'no-cache', 'Pragma': 'no-cache' }
61 req = urllib2.Request(url=src_url, headers=headers)
62 f = urllib2.urlopen(req)
64 except urllib2.HTTPError, error:
66 except urllib2.URLError, error:
68 if error.errno in [-3, 60, 61, 110, 111]:
69 b.log_line("unable to connect... trying again")
78 src_url = config.control_url + "/srpms/" + r.id + "/" + b.src_rpm
79 b.log_line("fetching %s" % src_url)
84 headers = { 'Cache-Control': 'no-cache', 'Pragma': 'no-cache' }
85 req = urllib2.Request(url=src_url, headers=headers)
86 f = urllib2.urlopen(req)
88 except urllib2.HTTPError, error:
89 # fail in a way where cron job will retry
90 msg = "unable to fetch file, http code: %d" % error.code
92 queue_time = time.time() - r.time
94 if error.code != 404 or (queue_time >= 0 and queue_time < (6 * 60 * 60)):
97 msg = "in queue for more than 6 hours, download failing"
100 except urllib2.URLError, error:
102 if error.errno in [-3, 60, 61, 110, 111]:
103 b.log_line("unable to connect... trying again")
108 o = chroot.popen("cat > %s" % b.src_rpm, mode = "w")
111 bytes = util.sendfile(f, o)
113 b.log_line("error: unable to write to `%s': %s" % (b.src_rpm, e))
118 t = time.time() - start
120 b.log_line("fetched %d bytes" % bytes)
122 b.log_line("fetched %d bytes, %.1f K/s" % (bytes, bytes / 1024.0 / t))
125 chroot.run("test ! -f /proc/uptime && mount /proc", 'root')
126 chroot.run("test ! -c /dev/full && rm -f /dev/full && mknod -m 666 /dev/full c 1 7", 'root')
127 chroot.run("test ! -c /dev/null && rm -f /dev/null && mknod -m 666 /dev/null c 1 3", 'root')
128 chroot.run("test ! -c /dev/random && rm -f /dev/random && mknod -m 644 /dev/random c 1 8", 'root')
129 chroot.run("test ! -c /dev/urandom && rm -f /dev/urandom && mknod -m 644 /dev/urandom c 1 9", 'root')
130 chroot.run("test ! -c /dev/zero && rm -f /dev/zero && mknod -m 666 /dev/zero c 1 5", 'root')
131 # make neccessary files readable for builder user
132 # TODO: see if they really aren't readable for builder
133 chroot.run("for db in Packages Name Basenames Providename Pubkey; do db=/var/lib/rpm/$db; chmod a+r $db; done", 'root')
134 # try to limit network access for builder account
135 chroot.run("/bin/setfacl -m u:builder:--- /etc/resolv.conf", 'root')
138 status.push("building %s" % b.spec)
139 b.log_line("request from: %s" % r.requester)
141 if check_skip_build(r, b):
142 b.log_line("build skipped due to src builder request")
143 res = "SKIP_REQUESTED"
146 b.log_line("started at: %s" % time.asctime())
148 b.log_line("installing srpm: %s" % b.src_rpm)
149 res = chroot.run("rpm -U %s" % b.src_rpm, logfile = b.logfile)
150 chroot.run("rm -f %s" % b.src_rpm, logfile = b.logfile)
152 tmpdir = "/tmp/B." + b.b_id[0:6]
154 b.log_line("error: installing src rpm failed")
155 res = "FAIL_SRPM_INSTALL"
158 chroot.run("install -m 700 -d %s" % tmpdir)
160 b.default_target(config.arch)
161 rpmbuild_opt = "%s %s %s" % (b.target_string(), b.kernel_string(), b.bconds_string())
162 # check for build arch before filling BR
163 cmd = "cd rpm/SPECS; TMPDIR=%s nice -n %s rpmbuild -bp --short-circuit --nodeps --define 'prep exit 0' %s %s" % \
164 (tmpdir, config.nice, rpmbuild_opt, b.spec)
165 res = chroot.run(cmd, logfile = b.logfile)
168 b.log_line("error: build arch check (%s) failed" % cmd)
171 if ("no-install-br" not in r.flags) and not install.uninstall_self_conflict(b):
172 res = "FAIL_DEPS_UNINSTALL"
173 if ("no-install-br" not in r.flags) and not install.install_br(r, b):
174 res = "FAIL_DEPS_INSTALL"
176 max_jobs = max(min(int(os.sysconf('SC_NPROCESSORS_ONLN') * 1.5), config.max_jobs), 1)
178 max_jobs = max(min(config.max_jobs, r.max_jobs), 1)
179 cmd = "cd rpm/SPECS; TMPDIR=%s nice -n %s rpmbuild -bb --define '_smp_mflags -j%d' %s %s" % \
180 (tmpdir, config.nice, max_jobs, rpmbuild_opt, b.spec)
181 b.log_line("building RPM using: %s" % cmd)
182 begin_time = time.time()
183 res = chroot.run(cmd, logfile = b.logfile)
184 end_time = time.time()
185 b.log_line("ended at: %s, done in %s" % (time.asctime(), time.strftime("%Hh %Mm %Ss", time.gmtime(end_time - begin_time))))
188 files = util.collect_files(b.logfile)
190 r.chroot_files.extend(files)
192 b.log_line("error: No files produced.")
193 last_section = util.find_last_section(b.logfile)
194 if last_section == None:
197 res = "FAIL_%s" % last_section.upper()
200 chroot.run("rm -rf %s; cd rpm/SPECS; rpmbuild --nodeps --nobuild " \
201 "--clean --rmspec --rmsource %s" % \
202 (tmpdir, b.spec), logfile = b.logfile)
203 chroot.run("rm -rf $HOME/rpm/BUILD/*")
206 util.append_to(b.logfile, l)
209 rpm_cache_dir = config.rpm_cache_dir
210 if "test-build" not in r.flags:
211 # NOTE: copying to cache dir doesn't mean that build failed, so ignore result
212 b.log_line("copy rpm files to cache_dir: %s" % rpm_cache_dir)
214 "cp -f %s %s && poldek --mo=nodiff --mkidxz -s %s/" % \
215 (string.join(b.files), rpm_cache_dir, rpm_cache_dir),
216 logfile = b.logfile, user = "root"
219 ll("test-build: not copying to " + rpm_cache_dir)
220 ll("Begin-PLD-Builder-Info")
221 if "upgrade" in r.flags:
222 b.upgraded = install.upgrade_from_batch(r, b)
225 ll("End-PLD-Builder-Info")
228 local = r.tmp_dir + os.path.basename(f)
229 chroot.cp(f, outfile = local, rm = True)
233 c="file:SRPMS:%s\n" % b.src_rpm
235 c=c + "file:ARCH:%s\n" % os.path.basename(f)
239 if config.gen_upinfo and b.files != [] and 'test-build' not in r.flags:
240 fname = r.tmp_dir + b.src_rpm + ".uploadinfo"
242 f.write(uploadinfo(b))
244 ftp.add(fname, "uploadinfo")
250 def handle_request(r):
253 build.build_all(r, build_rpm)
254 report.send_report(r, is_src = False)
261 f = open("/proc/loadavg")
262 if float(string.split(f.readline())[2]) > config.max_load:
269 def main_for(builder):
274 q = B_Queue(path.queue_file + "-" + config.builder)
280 req = pick_request(q)
283 # high priority tasks have priority < 0, normal tasks >= 0
284 if req.priority >= 0:
286 # allow only one build in given builder at once
287 if not lock.lock("building-rpm-for-%s" % config.builder, non_block = 1):
291 # not more then job_slots builds at once
293 for slot in range(config.job_slots):
294 if lock.lock("building-rpm-slot-%d" % slot, non_block = 1):
300 # record fact that we got lock for this builder, load balancer
301 # will use it for fair-queuing
302 l = lock.lock("got-lock")
303 f = open(path.got_lock_file, "a")
304 f.write(config.builder + "\n")
308 msg = "HIGH PRIORITY: "
310 msg += "handling request %s (%d) for %s from %s, priority %s" \
311 % (req.id, req.no, config.builder, req.requester, req.priority)
323 q = B_Queue(path.queue_file + "-" + config.builder)
326 previouslen=len(q.requests)
327 q.requests=filter(otherreqs, q.requests)
328 if len(q.requests)<previouslen:
333 if len(sys.argv) < 2:
334 raise Exception, "fatal: need to have builder name as first arg"
335 return main_for(sys.argv[1])
337 if __name__ == '__main__':