import os
import atexit
import time
+import datetime
import string
import urllib
+import urllib2
from config import config, init_conf
from bqueue import B_Queue
ret = q.requests[0]
return ret
+def check_skip_build(r, b):
+ src_url = config.control_url + "/srpms/" + r.id + "/skipme"
+ good = False
+ b.log_line("checking if we should skip the build")
+ while not good:
+ try:
+ headers = { 'Cache-Control': 'no-cache', 'Pragma': 'no-cache' }
+ req = urllib2.Request(url=src_url, headers=headers)
+ f = urllib2.urlopen(req)
+ good = True
+ except urllib2.HTTPError, error:
+ return False
+ except urllib2.URLError, error:
+ # see errno.h
+ try:
+ errno = error.errno
+ except AttributeError:
+ # python 2.4
+ errno = error.reason[0]
+
+ if errno in [-3, 60, 61, 110, 111]:
+ b.log_line("unable to connect... trying again")
+ continue
+ else:
+ return False
+ f.close()
+ return True
+ return False
+
def fetch_src(r, b):
- src_url = config.control_url + "/srpms/" + r.id + "/" + b.src_rpm
+ src_url = config.control_url + "/srpms/" + r.id + "/" + urllib.quote(b.src_rpm)
b.log_line("fetching %s" % src_url)
start = time.time()
- good=False
+ good = False
while not good:
try:
- good=True
- f = urllib.urlopen(src_url)
- except IOError, error:
- if error[1][0] == 60 or error[1][0] == 110 or error[1][0] == -3 or error[1][0] == 111 or error[1][0] == 61:
- good=False
- b.log_line("unable to connect... trying again")
+ headers = { 'Cache-Control': 'no-cache', 'Pragma': 'no-cache' }
+ req = urllib2.Request(url=src_url, headers=headers)
+ f = urllib2.urlopen(req)
+ good = True
+ except urllib2.HTTPError, error:
+ # fail in a way where cron job will retry
+ msg = "unable to fetch url %s, http code: %d" % (src_url, error.code)
+ b.log_line(msg)
+ queue_time = time.time() - r.time
+ # 6 hours
+ if error.code != 404 or (queue_time >= 0 and queue_time < (6 * 60 * 60)):
+ raise IOError, msg
else:
- f = urllib.urlopen(src_url) # So we get the exception logged :)
+ msg = "in queue for more than 6 hours, download failing"
+ b.log_line(msg)
+ return False
+ except urllib2.URLError, error:
+ errno = 0
+ if isinstance(error.args[0], IOError):
+ errno = error.args[0].errno
+
+ if errno in [-3, 60, 61, 110, 111]:
+ b.log_line("unable to connect to %s... trying again" % (src_url))
+ continue
+ else:
+ try:
+ print "error.errno: %s" % str(error.errno)
+ except Exception, e:
+ print "error.errno: exception %s" % e
+ try:
+ print "error.reason %s" % str(error.reason)
+ except Exception, e:
+ print "error.reason exception %s" % e
+ raise
o = chroot.popen("cat > %s" % b.src_rpm, mode = "w")
else:
b.log_line("fetched %d bytes, %.1f K/s" % (bytes, bytes / 1024.0 / t))
-def prepare_env():
- chroot.run("test ! -f /proc/uptime && mount /proc", 'root')
- chroot.run("test ! -e /dev/full && mknod /dev/full c 1 7 && chmod 666 /dev/full", 'root')
- chroot.run("test ! -e /dev/null && mknod /dev/null c 1 3 && chmod 666 /dev/null", 'root')
- chroot.run("test ! -e /dev/random && mknod /dev/random c 1 8 && chmod 644 /dev/random", 'root')
- chroot.run("test ! -e /dev/urandom && mknod /dev/urandom c 1 9 && chmod 644 /dev/urandom", 'root')
- chroot.run("test ! -e /dev/zero && mknod /dev/zero c 1 5 && chmod 666 /dev/zero", 'root')
+def prepare_env(logfile = None):
+ chroot.run("""
+ test ! -f /proc/uptime && mount /proc 2>/dev/null
+ test ! -c /dev/full && rm -f /dev/full && mknod -m 666 /dev/full c 1 7
+ test ! -c /dev/null && rm -f /dev/null && mknod -m 666 /dev/null c 1 3
+ test ! -c /dev/random && rm -f /dev/random && mknod -m 644 /dev/random c 1 8
+ test ! -c /dev/urandom && rm -f /dev/urandom && mknod -m 644 /dev/urandom c 1 9
+ test ! -c /dev/zero && rm -f /dev/zero && mknod -m 666 /dev/zero c 1 5
+
+ # need entry for "/" in mtab, for diskspace() to work in rpm
+ [ -z $(awk '$2 == "/" {print $1; exit}' /etc/mtab) ] && mount -f -t rootfs rootfs /
+
+ # make neccessary files readable for builder user
+ # TODO: see if they really aren't readable for builder
+ for db in Packages Name Basenames Providename Pubkeys; do
+ db=/var/lib/rpm/$db
+ test -f $db && chmod a+r $db
+ done
+
+ # try to limit network access for builder account
+ /bin/setfacl -m u:builder:--- /etc/resolv.conf
+ """, 'root', logfile = logfile)
def build_rpm(r, b):
- status.push("building %s" % b.spec)
+ packagename = b.get_package_name()
+ if not packagename:
+ # should not really get here
+ b.log_line("error: No .spec not given of malformed: '%s'" % b.spec)
+ res = "FAIL_INTERNAL"
+ return res
+
+ status.push("building %s (%s)" % (b.spec, packagename))
b.log_line("request from: %s" % r.requester)
+
+ if check_skip_build(r, b):
+ b.log_line("build skipped due to src builder request")
+ res = "SKIP_REQUESTED"
+ return res
+
b.log_line("started at: %s" % time.asctime())
fetch_src(r, b)
b.log_line("installing srpm: %s" % b.src_rpm)
- res = chroot.run("rpm -U %s" % b.src_rpm, logfile = b.logfile)
- chroot.run("rm -f %s" % b.src_rpm, logfile = b.logfile)
+ res = chroot.run("""
+ set -ex;
+ install -d %(topdir)s/{BUILD,RPMS};
+ LC_ALL=en_US.UTF-8 rpm -qp --changelog %(src_rpm)s;
+ rpm -Uhv --nodeps %(rpmdefs)s %(src_rpm)s;
+ rm -f %(src_rpm)s;
+ """ % {
+ 'topdir' : b._topdir,
+ 'rpmdefs' : b.rpmbuild_opts(),
+ 'src_rpm' : b.src_rpm
+ }, logfile = b.logfile)
b.files = []
- tmpdir = "/tmp/B." + b.b_id[0:6]
+
+ tmpdir = b.tmpdir()
if res:
b.log_line("error: installing src rpm failed")
res = "FAIL_SRPM_INSTALL"
else:
prepare_env()
- chroot.run("install -m 700 -d %s" % tmpdir)
-
+ chroot.run("set -x; install -m 700 -d %s" % tmpdir, logfile=b.logfile)
b.default_target(config.arch)
- rpmbuild_opt = "%s %s %s" % (b.target_string(), b.kernel_string(), b.bconds_string())
# check for build arch before filling BR
- cmd = "cd rpm/SPECS; TMPDIR=%s nice -n %s rpmbuild -bp --short-circuit --nodeps --define 'prep exit 0' %s %s" % \
- (tmpdir, config.nice, rpmbuild_opt, b.spec)
+ cmd = "set -ex; TMPDIR=%(tmpdir)s exec nice -n %(nice)s " \
+ "rpmbuild -bp --short-circuit --nodeps %(rpmdefs)s --define 'prep exit 0' %(topdir)s/%(spec)s" % {
+ 'tmpdir': tmpdir,
+ 'nice' : config.nice,
+ 'topdir' : b._topdir,
+ 'rpmdefs' : b.rpmbuild_opts(),
+ 'spec': b.spec,
+ }
res = chroot.run(cmd, logfile = b.logfile)
if res:
res = "UNSUPP"
if ("no-install-br" not in r.flags) and not install.install_br(r, b):
res = "FAIL_DEPS_INSTALL"
if not res:
- max_jobs = max(min(int(os.sysconf('SC_NPROCESSORS_ONLN') * 1.5), config.max_jobs), 1)
+ max_jobs = max(min(int(os.sysconf('SC_NPROCESSORS_ONLN') + 1), config.max_jobs), 1)
if r.max_jobs > 0:
max_jobs = max(min(config.max_jobs, r.max_jobs), 1)
- cmd = "cd rpm/SPECS; TMPDIR=%s nice -n %s rpmbuild -bb --define '_smp_mflags -j%d' %s %s" % \
- (tmpdir, config.nice, max_jobs, rpmbuild_opt, b.spec)
+ cmd = "set -ex; : build-id: %(r_id)s; TMPDIR=%(tmpdir)s exec nice -n %(nice)s " \
+ "rpmbuild -bb --define '_smp_mflags -j%(max_jobs)d' --define '_make_opts -Otarget' --define '_pld_builder 1' %(rpmdefs)s %(topdir)s/%(spec)s" % {
+ 'r_id' : r.id,
+ 'tmpdir': tmpdir,
+ 'nice' : config.nice,
+ 'rpmdefs' : b.rpmbuild_opts(),
+ 'topdir' : b._topdir,
+ 'max_jobs' : max_jobs,
+ 'spec': b.spec,
+ }
b.log_line("building RPM using: %s" % cmd)
begin_time = time.time()
res = chroot.run(cmd, logfile = b.logfile)
end_time = time.time()
- b.log_line("ended at: %s, done in %s" % (time.asctime(), time.strftime("%Hh %Mm %Ss", time.gmtime(end_time - begin_time))))
+ b.log_line("ended at: %s, done in %s" % (time.asctime(), datetime.timedelta(0, end_time - begin_time)))
if res:
res = "FAIL"
- files = util.collect_files(b.logfile)
+ files = util.collect_files(b.logfile, basedir = b._topdir)
if len(files) > 0:
r.chroot_files.extend(files)
else:
res = "FAIL_%s" % last_section.upper()
b.files = files
- chroot.run("rm -rf %s; cd rpm/SPECS; rpmbuild --nodeps --nobuild " \
- "--clean --rmspec --rmsource %s" % \
- (tmpdir, b.spec), logfile = b.logfile)
- chroot.run("rm -rf $HOME/rpm/BUILD/*")
+ # cleanup tmp and build files
+ chroot.run("""
+ set -ex;
+ chmod -R u+rwX %(topdir)s/BUILD;
+ rm -rf %(topdir)s/{tmp,BUILD}
+ """ % {
+ 'topdir' : b._topdir,
+ }, logfile = b.logfile)
def ll(l):
util.append_to(b.logfile, l)
-
+
if b.files != []:
rpm_cache_dir = config.rpm_cache_dir
if "test-build" not in r.flags:
- # XXX missing error check!
- b.log_line("copy rpm files to cache_dir=%s" % rpm_cache_dir)
- res = chroot.run(
+ # NOTE: copying to cache dir doesn't mean that build failed, so ignore result
+ b.log_line("copy rpm files to cache_dir: %s" % rpm_cache_dir)
+ chroot.run(
"cp -f %s %s && poldek --mo=nodiff --mkidxz -s %s/" % \
(string.join(b.files), rpm_cache_dir, rpm_cache_dir),
logfile = b.logfile, user = "root"
chroot.cp(f, outfile = local, rm = True)
ftp.add(local)
+ # cleanup all remains from this build
+ chroot.run("""
+ set -ex;
+ rm -rf %(topdir)s;
+ """ % {
+ 'topdir' : b._topdir,
+ }, logfile = b.logfile)
+
def uploadinfo(b):
c="file:SRPMS:%s\n" % b.src_rpm
for f in b.files:
f.close()
l.close()
else:
+ # be able to avoid locking with very low priority
+ if req.priority > -1000:
+ # don't kill server
+ check_load()
+ # allow only one build in given builder at once
+ if not lock.lock("building-high-priority-rpm-for-%s" % config.builder, non_block = 1):
+ return
+
msg = "HIGH PRIORITY: "
-
+
msg += "handling request %s (%d) for %s from %s, priority %s" \
% (req.id, req.no, config.builder, req.requester, req.priority)
log.notice(msg)
if len(q.requests)<previouslen:
q.write()
q.unlock()
-
+
def main():
if len(sys.argv) < 2:
raise Exception, "fatal: need to have builder name as first arg"
return main_for(sys.argv[1])
-
+
if __name__ == '__main__':
loop.run_loop(main)
projects / projects / pld-builder.new.git / blobdiff