[projects/pld-builder.new.git] / PLD_Builder / rpm_builder.py

# vi: encoding=utf-8 ts=8 sts=4 sw=4 et

import sys
import os
import atexit
import time
import string
import urllib

from config import config, init_conf
from bqueue import B_Queue
import lock
import util
import loop
import path
import status
import log
import chroot
import ftp
import buildlogs
import notify
import build
import report
import install

# *HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*
import socket

socket.myorigsocket=socket.socket

def mysocket(family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0):
    s=socket.myorigsocket(family, type, proto)
    s.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
    return s

socket.socket=mysocket
# *HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*

# this code is duplicated in srpm_builder, but we
# might want to handle some cases differently here
def pick_request(q):
    def mycmp(r1, r2):
        if r1.kind != 'group' or r2.kind != 'group':
            raise Exception, "non-group requests"
        pri_diff = cmp(r1.priority, r2.priority)
        if pri_diff == 0:
            return cmp(r1.time, r2.time)
        else:
            return pri_diff
    q.requests.sort(mycmp)
    ret = q.requests[0]
    return ret

def fetch_src(r, b):
    src_url = config.control_url + "/srpms/" + r.id + "/" + b.src_rpm
    b.log_line("fetching %s" % src_url)
    start = time.time()
    good=False
    while not good:
        try:
            good=True
            f = urllib.urlopen(src_url)
        except IOError, error:
            if error[1][0] == 60 or error[1][0] == 110 or error[1][0] == -3 or error[1][0] == 111 or error[1][0] == 61:
                good=False
                b.log_line("unable to connect... trying again")
            else:
                f = urllib.urlopen(src_url) # So we get the exception logged :)

    o = chroot.popen("cat > %s" % b.src_rpm, mode = "w")

    try:
        bytes = util.sendfile(f, o)
    except IOError, e:
        b.log_line("error: unable to write to `%s': %s" % (b.src_rpm, e))
        raise

    f.close()
    o.close()
    t = time.time() - start
    if t == 0:
        b.log_line("fetched %d bytes" % bytes)
    else:
        b.log_line("fetched %d bytes, %.1f K/s" % (bytes, bytes / 1024.0 / t))

def prepare_env():
    chroot.run("test ! -f /proc/uptime && mount /proc", 'root')
    chroot.run("test ! -e /dev/full && mknod /dev/full c 1 7 && chmod 666 /dev/full", 'root')
    chroot.run("test ! -e /dev/null && mknod /dev/null c 1 3 && chmod 666 /dev/null", 'root')
    chroot.run("test ! -e /dev/random && mknod /dev/random c 1 8 && chmod 644 /dev/random", 'root')
    chroot.run("test ! -e /dev/urandom && mknod /dev/urandom c 1 9 && chmod 644 /dev/urandom", 'root')
    chroot.run("test ! -e /dev/zero && mknod /dev/zero c 1 5 && chmod 666 /dev/zero", 'root')
    # try to limit network access for builder account
    chroot.run("/bin/setfacl -m u:builder:--- /etc/resolv.conf", 'root')

def build_rpm(r, b):
    status.push("building %s" % b.spec)
    b.log_line("request from: %s" % r.requester)
    b.log_line("started at: %s" % time.asctime())
    fetch_src(r, b)
    b.log_line("installing srpm: %s" % b.src_rpm)
    res = chroot.run("rpm -U %s" % b.src_rpm, logfile = b.logfile)
    chroot.run("rm -f %s" % b.src_rpm, logfile = b.logfile)
    b.files = []
    tmpdir = "/tmp/B." + b.b_id[0:6]
    if res:
        b.log_line("error: installing src rpm failed")
        res = "FAIL_SRPM_INSTALL"
    else:
        prepare_env()
        chroot.run("install -m 700 -d %s" % tmpdir)

        b.default_target(config.arch)
        rpmbuild_opt = "%s %s %s" % (b.target_string(), b.kernel_string(), b.bconds_string())
        # check for build arch before filling BR
        cmd = "cd rpm/SPECS; TMPDIR=%s nice -n %s rpmbuild -bp --short-circuit --nodeps --define 'prep exit 0' %s %s" % \
            (tmpdir, config.nice, rpmbuild_opt, b.spec)
        res = chroot.run(cmd, logfile = b.logfile)
        if res:
            res = "UNSUPP"
            b.log_line("error: build arch check (%s) failed" % cmd)

        if not res:
            if ("no-install-br" not in r.flags) and not install.uninstall_self_conflict(b):
                res = "FAIL_DEPS_UNINSTALL"
            if ("no-install-br" not in r.flags) and not install.install_br(r, b):
                res = "FAIL_DEPS_INSTALL"
            if not res:
                max_jobs = max(min(int(os.sysconf('SC_NPROCESSORS_ONLN') * 1.5), config.max_jobs), 1)
                if r.max_jobs > 0:
                    max_jobs = max(min(config.max_jobs, r.max_jobs), 1)
                cmd = "cd rpm/SPECS; TMPDIR=%s nice -n %s rpmbuild -bb --define '_smp_mflags -j%d' %s %s" % \
                            (tmpdir, config.nice, max_jobs, rpmbuild_opt, b.spec)
                b.log_line("building RPM using: %s" % cmd)
                begin_time = time.time()
                res = chroot.run(cmd, logfile = b.logfile)
                end_time = time.time()
                b.log_line("ended at: %s, done in %s" % (time.asctime(), time.strftime("%Hh %Mm %Ss", time.gmtime(end_time - begin_time))))
                if res:
                    res = "FAIL"
                files = util.collect_files(b.logfile)
                if len(files) > 0:
                    r.chroot_files.extend(files)
                else:
                    b.log_line("error: No files produced.")
                    last_section = util.find_last_section(b.logfile)
                    if last_section == None:
                        res = "FAIL"
                    else:
                        res = "FAIL_%s" % last_section.upper()
                b.files = files

    chroot.run("rm -rf %s; cd rpm/SPECS; rpmbuild --nodeps --nobuild " \
                         "--clean --rmspec --rmsource %s" % \
                         (tmpdir, b.spec), logfile = b.logfile)
    chroot.run("rm -rf $HOME/rpm/BUILD/*")

    def ll(l):
        util.append_to(b.logfile, l)
 
    if b.files != []:
        rpm_cache_dir = config.rpm_cache_dir
        if "test-build" not in r.flags:
            # XXX missing error check!
            b.log_line("copy rpm files to cache_dir=%s" % rpm_cache_dir)
            res = chroot.run(
                    "cp -f %s %s && poldek --mo=nodiff --mkidxz -s %s/" % \
                        (string.join(b.files), rpm_cache_dir, rpm_cache_dir),
                     logfile = b.logfile, user = "root"
            )
        else:
            ll("test-build: not copying to " + rpm_cache_dir)
        ll("Begin-PLD-Builder-Info")
        if "upgrade" in r.flags:
            b.upgraded = install.upgrade_from_batch(r, b)
        else:
            ll("not upgrading")
        ll("End-PLD-Builder-Info")

    for f in b.files:
        local = r.tmp_dir + os.path.basename(f)
        chroot.cp(f, outfile = local, rm = True)
        ftp.add(local)

    def uploadinfo(b):
        c="file:SRPMS:%s\n" % b.src_rpm
        for f in b.files:
            c=c + "file:ARCH:%s\n" % os.path.basename(f)
        c=c + "END\n"
        return c

    if config.gen_upinfo and b.files != [] and 'test-build' not in r.flags:
        fname = r.tmp_dir + b.src_rpm + ".uploadinfo"
        f = open(fname, "w")
        f.write(uploadinfo(b))
        f.close()
        ftp.add(fname, "uploadinfo")

    status.pop()

    return res

def handle_request(r):
    ftp.init(r)
    buildlogs.init(r)
    build.build_all(r, build_rpm)
    report.send_report(r, is_src = False)
    ftp.flush()
    notify.send(r)

def check_load():
    do_exit = 0
    try:
        f = open("/proc/loadavg")
        if float(string.split(f.readline())[2]) > config.max_load:
            do_exit = 1
    except:
        pass
    if do_exit:
        sys.exit(0)

def main_for(builder):
    msg = ""

    init_conf(builder)

    q = B_Queue(path.queue_file + "-" + config.builder)
    q.lock(0)
    q.read()
    if q.requests == []:
        q.unlock()
        return
    req = pick_request(q)
    q.unlock()

    # high priority tasks have priority < 0, normal tasks >= 0
    if req.priority >= 0:

        # allow only one build in given builder at once
        if not lock.lock("building-rpm-for-%s" % config.builder, non_block = 1):
            return
        # don't kill server
        check_load()
        # not more then job_slots builds at once
        locked = 0
        for slot in range(config.job_slots):
            if lock.lock("building-rpm-slot-%d" % slot, non_block = 1):
                locked = 1
                break
        if not locked:
            return

        # record fact that we got lock for this builder, load balancer
        # will use it for fair-queuing
        l = lock.lock("got-lock")
        f = open(path.got_lock_file, "a")
        f.write(config.builder + "\n")
        f.close()
        l.close()
    else:
        msg = "HIGH PRIORITY: "
    
    msg += "handling request %s (%d) for %s from %s, priority %s" \
            % (req.id, req.no, config.builder, req.requester, req.priority)
    log.notice(msg)
    status.push(msg)
    handle_request(req)
    status.pop()

    def otherreqs(r):
        if r.no==req.no:
            return False
        else:
            return True

    q = B_Queue(path.queue_file + "-" + config.builder)
    q.lock(0)
    q.read()
    previouslen=len(q.requests)
    q.requests=filter(otherreqs, q.requests)
    if len(q.requests)<previouslen:
        q.write()
    q.unlock()
    
def main():
    if len(sys.argv) < 2:
        raise Exception, "fatal: need to have builder name as first arg"
    return main_for(sys.argv[1])
    
if __name__ == '__main__':
    loop.run_loop(main)
Commit	Line	Data
dfff8bd5 MM	1	# vi: encoding=utf-8 ts=8 sts=4 sw=4 et
dfff8bd5 MM	2
3ea4b156 MM	3	import sys
	4	import os
	5	import atexit
	6	import time
deda9a51	7	import string
3ea4b156 MM	8	import urllib
	9
	10	from config import config, init_conf
	11	from bqueue import B_Queue
3ea4b156 MM	12	import lock
3ea4b156 MM	13	import util
3f446d8f	14	import loop
3ea4b156 MM	15	import path
	16	import status
	17	import log
	18	import chroot
	19	import ftp
e2cad913	20	import buildlogs
59ce7cd6	21	import notify
17f23d66 MM	22	import build
17f23d66 MM	23	import report
73bdb36b	24	import install
3ea4b156	25
53989cf3 MM	26	# HACKHACKHACKHACKHACKHACKHACKHACKHACKHACKHACKHACKHACKHACKHACK
	27	import socket
	28
	29	socket.myorigsocket=socket.socket
	30
	31	def mysocket(family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0):
dfff8bd5 MM	32	s=socket.myorigsocket(family, type, proto)
	33	s.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
	34	return s
53989cf3 MM	35
	36	socket.socket=mysocket
	37	# HACKHACKHACKHACKHACKHACKHACKHACKHACKHACKHACKHACKHACKHACKHACK
	38
3ea4b156 MM	39	# this code is duplicated in srpm_builder, but we
	40	# might want to handle some cases differently here
	41	def pick_request(q):
dfff8bd5 MM	42	def mycmp(r1, r2):
dfff8bd5 MM	43	if r1.kind != 'group' or r2.kind != 'group':
939af6d7	44	raise Exception, "non-group requests"
dfff8bd5 MM	45	pri_diff = cmp(r1.priority, r2.priority)
	46	if pri_diff == 0:
	47	return cmp(r1.time, r2.time)
	48	else:
bbed4c4d	49	return pri_diff
dfff8bd5 MM	50	q.requests.sort(mycmp)
	51	ret = q.requests[0]
	52	return ret
3ea4b156	53
17f23d66	54	def fetch_src(r, b):
dfff8bd5 MM	55	src_url = config.control_url + "/srpms/" + r.id + "/" + b.src_rpm
	56	b.log_line("fetching %s" % src_url)
	57	start = time.time()
	58	good=False
	59	while not good:
	60	try:
	61	good=True
	62	f = urllib.urlopen(src_url)
	63	except IOError, error:
	64	if error[1][0] == 60 or error[1][0] == 110 or error[1][0] == -3 or error[1][0] == 111 or error[1][0] == 61:
	65	good=False
	66	b.log_line("unable to connect... trying again")
	67	else:
	68	f = urllib.urlopen(src_url) # So we get the exception logged :)
	69
	70	o = chroot.popen("cat > %s" % b.src_rpm, mode = "w")
9e89ce60 AM	71
	72	try:
	73	bytes = util.sendfile(f, o)
	74	except IOError, e:
00a1db68	75	b.log_line("error: unable to write to `%s': %s" % (b.src_rpm, e))
9e89ce60 AM	76	raise
9e89ce60 AM	77
dfff8bd5 MM	78	f.close()
	79	o.close()
	80	t = time.time() - start
	81	if t == 0:
	82	b.log_line("fetched %d bytes" % bytes)
	83	else:
	84	b.log_line("fetched %d bytes, %.1f K/s" % (bytes, bytes / 1024.0 / t))
3ea4b156	85
84c6175b MM	86	def prepare_env():
84c6175b MM	87	chroot.run("test ! -f /proc/uptime && mount /proc", 'root')
9b2ab0b4 AM	88	chroot.run("test ! -e /dev/full && mknod /dev/full c 1 7 && chmod 666 /dev/full", 'root')
	89	chroot.run("test ! -e /dev/null && mknod /dev/null c 1 3 && chmod 666 /dev/null", 'root')
	90	chroot.run("test ! -e /dev/random && mknod /dev/random c 1 8 && chmod 644 /dev/random", 'root')
	91	chroot.run("test ! -e /dev/urandom && mknod /dev/urandom c 1 9 && chmod 644 /dev/urandom", 'root')
	92	chroot.run("test ! -e /dev/zero && mknod /dev/zero c 1 5 && chmod 666 /dev/zero", 'root')
62aff600	93	# try to limit network access for builder account
52a64a7d	94	chroot.run("/bin/setfacl -m u:builder:--- /etc/resolv.conf", 'root')
84c6175b	95
17f23d66	96	def build_rpm(r, b):
dfff8bd5 MM	97	status.push("building %s" % b.spec)
	98	b.log_line("request from: %s" % r.requester)
	99	b.log_line("started at: %s" % time.asctime())
	100	fetch_src(r, b)
	101	b.log_line("installing srpm: %s" % b.src_rpm)
	102	res = chroot.run("rpm -U %s" % b.src_rpm, logfile = b.logfile)
	103	chroot.run("rm -f %s" % b.src_rpm, logfile = b.logfile)
	104	b.files = []
	105	tmpdir = "/tmp/B." + b.b_id[0:6]
	106	if res:
	107	b.log_line("error: installing src rpm failed")
511ece79	108	res = "FAIL_SRPM_INSTALL"
deda9a51	109	else:
84c6175b	110	prepare_env()
dfff8bd5	111	chroot.run("install -m 700 -d %s" % tmpdir)
be264f26 ER	112
be264f26 ER	113	b.default_target(config.arch)
2e5736d2	114	rpmbuild_opt = "%s %s %s" % (b.target_string(), b.kernel_string(), b.bconds_string())
81c135f6 ER	115	# check for build arch before filling BR
	116	cmd = "cd rpm/SPECS; TMPDIR=%s nice -n %s rpmbuild -bp --short-circuit --nodeps --define 'prep exit 0' %s %s" % \
	117	(tmpdir, config.nice, rpmbuild_opt, b.spec)
	118	res = chroot.run(cmd, logfile = b.logfile)
4f2d26fb	119	if res:
511ece79	120	res = "UNSUPP"
a957e64f	121	b.log_line("error: build arch check (%s) failed" % cmd)
81c135f6 ER	122
81c135f6 ER	123	if not res:
feb26777	124	if ("no-install-br" not in r.flags) and not install.uninstall_self_conflict(b):
511ece79	125	res = "FAIL_DEPS_UNINSTALL"
feb26777	126	if ("no-install-br" not in r.flags) and not install.install_br(r, b):
511ece79	127	res = "FAIL_DEPS_INSTALL"
73bdb36b	128	if not res:
a1b28a50	129	max_jobs = max(min(int(os.sysconf('SC_NPROCESSORS_ONLN') * 1.5), config.max_jobs), 1)
85303822	130	if r.max_jobs > 0:
a1b28a50	131	max_jobs = max(min(config.max_jobs, r.max_jobs), 1)
b3c8d962	132	cmd = "cd rpm/SPECS; TMPDIR=%s nice -n %s rpmbuild -bb --define '_smp_mflags -j%d' %s %s" % \
85303822	133	(tmpdir, config.nice, max_jobs, rpmbuild_opt, b.spec)
81c135f6	134	b.log_line("building RPM using: %s" % cmd)
efb3621d	135	begin_time = time.time()
81c135f6	136	res = chroot.run(cmd, logfile = b.logfile)
efb3621d	137	end_time = time.time()
200299dd	138	b.log_line("ended at: %s, done in %s" % (time.asctime(), time.strftime("%Hh %Mm %Ss", time.gmtime(end_time - begin_time))))
4f2d26fb AM	139	if res:
4f2d26fb AM	140	res = "FAIL"
81c135f6 ER	141	files = util.collect_files(b.logfile)
	142	if len(files) > 0:
	143	r.chroot_files.extend(files)
	144	else:
	145	b.log_line("error: No files produced.")
2b9e7381 AM	146	last_section = util.find_last_section(b.logfile)
	147	if last_section == None:
	148	res = "FAIL"
	149	else:
	150	res = "FAIL_%s" % last_section.upper()
81c135f6 ER	151	b.files = files
81c135f6 ER	152
dfff8bd5 MM	153	chroot.run("rm -rf %s; cd rpm/SPECS; rpmbuild --nodeps --nobuild " \
	154	"--clean --rmspec --rmsource %s" % \
	155	(tmpdir, b.spec), logfile = b.logfile)
	156	chroot.run("rm -rf $HOME/rpm/BUILD/*")
	157
	158	def ll(l):
	159	util.append_to(b.logfile, l)
deda9a51	160
dfff8bd5	161	if b.files != []:
fd499dc1	162	rpm_cache_dir = config.rpm_cache_dir
e936beda	163	if "test-build" not in r.flags:
fd499dc1 ER	164	# XXX missing error check!
	165	b.log_line("copy rpm files to cache_dir=%s" % rpm_cache_dir)
	166	res = chroot.run(
	167	"cp -f %s %s && poldek --mo=nodiff --mkidxz -s %s/" % \
	168	(string.join(b.files), rpm_cache_dir, rpm_cache_dir),
	169	logfile = b.logfile, user = "root"
	170	)
e936beda	171	else:
fd499dc1	172	ll("test-build: not copying to " + rpm_cache_dir)
dfff8bd5 MM	173	ll("Begin-PLD-Builder-Info")
dfff8bd5 MM	174	if "upgrade" in r.flags:
73bdb36b	175	b.upgraded = install.upgrade_from_batch(r, b)
dfff8bd5 MM	176	else:
	177	ll("not upgrading")
	178	ll("End-PLD-Builder-Info")
deda9a51	179
1051562e	180	for f in b.files:
dfff8bd5	181	local = r.tmp_dir + os.path.basename(f)
3e689257	182	chroot.cp(f, outfile = local, rm = True)
dfff8bd5	183	ftp.add(local)
1051562e	184
dfff8bd5 MM	185	def uploadinfo(b):
	186	c="file:SRPMS:%s\n" % b.src_rpm
	187	for f in b.files:
	188	c=c + "file:ARCH:%s\n" % os.path.basename(f)
	189	c=c + "END\n"
	190	return c
3ea4b156	191
71a06b82	192	if config.gen_upinfo and b.files != [] and 'test-build' not in r.flags:
dfff8bd5 MM	193	fname = r.tmp_dir + b.src_rpm + ".uploadinfo"
	194	f = open(fname, "w")
	195	f.write(uploadinfo(b))
	196	f.close()
	197	ftp.add(fname, "uploadinfo")
	198
	199	status.pop()
	200
	201	return res
3ea4b156	202
17f23d66	203	def handle_request(r):
dfff8bd5 MM	204	ftp.init(r)
	205	buildlogs.init(r)
	206	build.build_all(r, build_rpm)
	207	report.send_report(r, is_src = False)
	208	ftp.flush()
63319e0a	209	notify.send(r)
3ea4b156 MM	210
3ea4b156 MM	211	def check_load():
dfff8bd5 MM	212	do_exit = 0
	213	try:
	214	f = open("/proc/loadavg")
	215	if float(string.split(f.readline())[2]) > config.max_load:
	216	do_exit = 1
	217	except:
	218	pass
	219	if do_exit:
	220	sys.exit(0)
3ea4b156	221
e8ee9db8	222	def main_for(builder):
b8ec18af AM	223	msg = ""
b8ec18af AM	224
dfff8bd5	225	init_conf(builder)
dfff8bd5	226
dfff8bd5 MM	227	q = B_Queue(path.queue_file + "-" + config.builder)
	228	q.lock(0)
	229	q.read()
	230	if q.requests == []:
	231	q.unlock()
	232	return
	233	req = pick_request(q)
7b16924c	234	q.unlock()
dfff8bd5	235
91112237 AM	236	# high priority tasks have priority < 0, normal tasks >= 0
91112237 AM	237	if req.priority >= 0:
b8ec18af AM	238
	239	# allow only one build in given builder at once
	240	if not lock.lock("building-rpm-for-%s" % config.builder, non_block = 1):
	241	return
	242	# don't kill server
	243	check_load()
	244	# not more then job_slots builds at once
	245	locked = 0
	246	for slot in range(config.job_slots):
	247	if lock.lock("building-rpm-slot-%d" % slot, non_block = 1):
	248	locked = 1
	249	break
	250	if not locked:
	251	return
	252
	253	# record fact that we got lock for this builder, load balancer
	254	# will use it for fair-queuing
	255	l = lock.lock("got-lock")
	256	f = open(path.got_lock_file, "a")
	257	f.write(config.builder + "\n")
	258	f.close()
	259	l.close()
	260	else:
	261	msg = "HIGH PRIORITY: "
dfff8bd5	262
b8ec18af AM	263	msg += "handling request %s (%d) for %s from %s, priority %s" \
b8ec18af AM	264	% (req.id, req.no, config.builder, req.requester, req.priority)
dfff8bd5 MM	265	log.notice(msg)
	266	status.push(msg)
	267	handle_request(req)
	268	status.pop()
	269
	270	def otherreqs(r):
	271	if r.no==req.no:
	272	return False
	273	else:
	274	return True
	275
	276	q = B_Queue(path.queue_file + "-" + config.builder)
	277	q.lock(0)
	278	q.read()
	279	previouslen=len(q.requests)
	280	q.requests=filter(otherreqs, q.requests)
	281	if len(q.requests)<previouslen:
	282	q.write()
	283	q.unlock()
	284
e8ee9db8	285	def main():
dfff8bd5	286	if len(sys.argv) < 2:
939af6d7	287	raise Exception, "fatal: need to have builder name as first arg"
dfff8bd5 MM	288	return main_for(sys.argv[1])
dfff8bd5 MM	289
e8ee9db8	290	if __name__ == '__main__':
dfff8bd5	291	loop.run_loop(main)