self.privs = []
self.gpg_emails = []
self.mailto = ""
-
+
if p.has_option(login, "gpg_emails"):
self.gpg_emails = string.split(p.get(login, "gpg_emails"))
else:
log.panic("acl: [%s] has no gpg_emails" % login)
-
+
if p.has_option(login, "mailto"):
self.mailto = p.get(login, "mailto")
else:
if len(self.gpg_emails) > 0:
self.mailto = self.gpg_emails[0]
-
+
if p.has_option(login, "privs"):
for p in string.split(p.get(login, "privs")):
l = string.split(p, ":")
self.users[e] = user
self.users[login] = user
status.pop()
-
+
def user_by_email(self, ems):
for e in ems:
if self.users.has_key(e):
self.fd = open(self.name, "r+")
else:
self.fd = open(self.name, "w+")
-
+
def read(self):
self._open()
self.signers = []
return 1
except IOError:
return 0
-
+
def unlock(self):
fcntl.flock(self.fd, fcntl.LOCK_UN)
if dep.build_failed:
can_build = 0
failed_dep = dep.spec
-
+
if batch.is_command() and can_build:
batch.logfile = tmp + "command"
if config.builder in batch.builders:
batch.build_time = ""
log.notice("building %s %s" % (batch.spec, batch.skip_reason))
notify.add_batch(batch, "SKIP")
-
+
buildlogs.flush()
chroot.run("rm -f %s" % string.join(r.chroot_files))
Requester: %s
END
""" % (config.buildlogs_url, l['name'], config.builder, time.time(), self.requester_email)
-
+
q = self.queue[:]
for l in q:
f = open(path.buildlogs_queue_dir + '/' + l['id'] + ".desc.tmp", "w")
def quote(cmd):
return re.sub("([\"\\\\$`])", r"\\\1", cmd)
-
+
def command(cmd, user = None, nostdin=""):
if user == None:
user = config.builder_user
nostdin = "exec < /dev/null; "
return "%s sudo chroot %s su - %s -c \"export LC_ALL=C; %s %s\"" \
% (config.sudo_chroot_wrapper, config.chroot, user, nostdin, quote(cmd))
-
+
def command_sh(cmd):
return "%s sudo chroot %s /bin/sh -c \"export LC_ALL=C; exec < /dev/null; %s\"" \
% (config.sudo_chroot_wrapper, config.chroot, quote(cmd))
def popen(cmd, user = "builder", mode = "r"):
f = os.popen(command(cmd, user), mode)
return f
-
+
def run(cmd, user = "builder", logfile = None, logstdout = None):
c = command(cmd, user, nostdin=True)
if logfile != None:
return d
else:
log.panic("cannot find %s::%s" % (builder, o))
-
+
p.readfp(open(path.builder_conf))
self.admin_email = get("admin_email")
self.builder_user = get("builder_user", "builder")
self.sudo_chroot_wrapper = get("sudo_chroot_wrapper", "")
self.nice = get("nice", "0")
-
+
f = get("syslog", "")
if f != "":
if syslog_facilities.has_key(f):
rpm_prov = {}
# list of required files
req_files = {}
-
+
def get_req():
msg("rpm-req... ")
f = chr_popen("rpm -qa --qf '@\n%{NAME}\n[%{REQUIRENAME}\n]'")
msg("[%s: %s, %s] " % (what, rpm_prov[what], pkg))
else:
rpm_prov[what] = pkg
-
+
def get_prov():
msg("rpm-prov... ")
f = chr_popen("rpm -qa --qf '@\n%{NAME}\n[%{PROVIDENAME}\n]'")
del req_files[l]
f.close()
msg("done\n")
-
+
def get_prov_files():
msg("rpm-files... ")
f = chr_popen("rpm -qa --qf '@\n%{NAME}\n[%{FILENAMES}\n]'")
msg("[%s: %s] " % (pkg, req))
requires[pkg] = pkg_reqs
msg("done\n")
-
+
# map from pkg-name to list of pkg-names required by it
# this is result
requires = {}
def remove_list(req, need):
"""List of packages scheduled for removal.
-
+
Given dependency information and list of needed packages compute list
of packages that don't need to be present.
"""
os.unlink(dir + "/retry-at")
except:
pass
-
+
status.push("flushing %s" % dir)
if flush_queue(dir):
else:
return rc
q.sort(mycmp)
-
+
error = None
# copy of q
remaining = q[:]
os.unlink(d['_file'])
os.unlink(d['_desc'])
remaining.remove(d)
-
+
if error != None:
emails = {}
emails[config.admin_email] = 1
emails[d['Requester']] = 1
e = emails.keys()
m = mailer.Message()
- m.set_headers(to = string.join(e, ", "),
+ m.set_headers(to = string.join(e, ", "),
subject = "[%s] builder queue problem" % config.builder)
m.write("there were problems sending files from queue %s:\n" % dir)
m.write("problems:\n")
self.url = config.test_ftp_url
else:
self.url = config.ftp_url
-
+
def add(self, file, type):
# if /dev/null, say bye bye
if self.url == "/dev/null":
Requester: %s
END
""" % (self.url, l['name'], config.builder, time.time(), l['type'], self.requester_email)
-
+
for l in self.queue:
f = open(path.ftp_queue_dir + '/' + l['id'] + ".desc", "w")
f.write(desc(l))
def flush():
queue.flush()
-
+
def kill():
queue.kill()
def status():
return queue.status
-
+
def clear_status():
queue.status = ""
def get_build_requires(spec, bconds_with, bconds_without):
cond_rx = re.compile(r"%\{(\!\?|\?\!|\?)([a-zA-Z0-9_+]+)\s*:([^%\{\}]*)\}")
-
+
def expand_conds(l):
def expand_one(m):
if m.group(1) == "?":
if not macros.has_key(m.group(2)):
return m.group(3)
return ""
-
+
for i in range(10):
l = cond_rx.sub(expand_one, l)
if len(l) > 1000: break
return string.strip(macros[m.group(1)])
else:
return m.group(0) # don't change
-
+
for i in range(10):
l = macro_rx.sub(expand_one, l)
if len(l) > 1000: break
-
+
return expand_conds(l)
-
+
simple_br_rx = re.compile(r"^BuildRequires\s*:\s*([^\s]+)", re.I)
bcond_rx = re.compile(r"^%bcond_(with|without)\s+([^\s]+)")
version_rx = re.compile(r"^Version\s*:\s*([^\s]+)", re.I)
name_rx = re.compile(r"^Name\s*:\s*([^\s]+)", re.I)
define_rx = re.compile(r"^\%define\s+([a-zA-Z0-9_+]+)\s+(.*)", re.I)
any_br_rx = re.compile(r"BuildRequires", re.I)
-
+
macros = {}
for b in bconds_with:
macros["_with_%s" % b] = 1
macros["kgcc_package"] = "gcc"
build_req = []
-
+
f = open(spec)
for l in xreadlines.xreadlines(f):
l = string.strip(l)
if l == "%changelog": break
-
+
# %bcond_with..
m = bcond_rx.search(l)
if m:
bcond = m.group(2)
if m.group(1) == "with":
- if macros.has_key("_with_%s" % bcond):
+ if macros.has_key("_with_%s" % bcond):
macros["with_%s" % bcond] = 1
else:
- if not macros.has_key("_without_%s" % bcond):
+ if not macros.has_key("_without_%s" % bcond):
macros["with_%s" % bcond] = 1
continue
-
+
# name,version,release
m = version_rx.search(l)
if m: macros["version"] = m.group(1)
# %define
m = define_rx.search(l)
if m: macros[m.group(1)] = m.group(2)
-
+
# *BuildRequires*
if any_br_rx.search(l):
l = expand_macros(l)
rx = re.compile("^gpg: Signature made .*using [DR]SA key ID (.+)")
keys = []
-
+
for l in d_stderr.split('\n'):
m = rx.match(l)
if m:
if len(needed) == 0:
b.log_line("no BR needed")
return True
-
+
nbr = ""
for bre in needed.keys():
nbr = nbr + " " + re.escape(bre)
for b in config.binary_builders:
bs[b] = 0
bl.append(b)
-
+
lck = lock.lock("got-lock")
f = open(path.got_lock_file, "r+")
line_no = 0
-
+
for l in f.xreadlines():
line_no += 1
b = string.strip(l)
def mycmp(b1, b2):
return cmp(bs[b1], bs[b2])
-
+
bl.sort(mycmp)
f.seek(0)
f = open(path.log_file, "a")
f.write("%s [%s]: %s [%s]\n" % (time.asctime(), builder, s, os.path.basename(sys.argv[0])))
f.close()
-
+
def panic(s):
log(syslog.LOG_ALERT, "PANIC: %s" % s)
raise Exception, "PANIC: %s" % str(s)
def alert(s):
- log(syslog.LOG_ALERT, "alert: %s" % s)
-
+ log(syslog.LOG_ALERT, "alert: %s" % s)
+
def error(s):
- log(syslog.LOG_ERR, "error: %s" % s)
-
+ log(syslog.LOG_ERR, "error: %s" % s)
+
def warn(s):
- log(syslog.LOG_WARNING, "warning: %s" % s)
-
+ log(syslog.LOG_WARNING, "warning: %s" % s)
+
def notice(s):
- log(syslog.LOG_NOTICE, "notice: %s" % s)
+ log(syslog.LOG_NOTICE, "notice: %s" % s)
def open_syslog(name, f):
global do_syslog
sys.exit(s)
else:
sys.exit(10)
-
+
start = time.time()
while time.time() - start < max:
last = time.time()
self.xml = StringIO.StringIO()
self.xml.write("<notification group-id='%s' builder='%s'>\n" % \
(g.id, config.builder))
-
+
def send(self, r):
sio = StringIO.StringIO()
self.xml.write("</notification>\n")
notifyq.init(r)
notifyq.add(sio)
notifyq.flush()
-
+
def add_batch(self, b, s):
self.xml.write(" <batch id='%s' status='%s' />\n" % (b.b_id, s))
-
+
n = None
def begin(group):
self.queue = []
self.requester_email = g.requester_email
self.notify_url = config.notify_url
-
+
def add(self, file):
id = util.uuid()
f = open(path.notify_queue_dir + '/' + id, 'w')
Requester: %s
END
""" % (self.notify_url, l['id'], config.builder, time.time(), self.requester_email)
-
+
for l in self.queue:
f = open(path.notify_queue_dir + '/' + l['id'] + ".desc", "w")
f.write(desc(l))
def flush():
queue.flush()
-
+
def kill():
queue.kill()
def status():
return queue.status
-
+
def clear_status():
queue.status = ""
# and intersection method on set object
def intersect(a, b):
r = []
- for x in a:
+ for x in a:
if x in b: r.append(x)
return r
-
+
# add given req-list to cur_pkg_reqs
def add_req(reqs):
if len(reqs) == 1:
return
if not did:
cur_pkg_reqs.append(reqs)
-
+
pkg_reqs = {}
cur_pkg_reqs = None
cur_pkg = None
-
+
f = chr_popen("poldek -v -v --verify --unique-pkg-names")
for l in xreadlines.xreadlines(f):
m = name_rx.match(l)
else:
msg("poldek_reqs: bad pkg name: %s\n" % x)
if len(reqs) != 0: add_req(reqs)
-
+
f.close()
if cur_pkg:
copy_mode = 0
else:
target.write(l)
-
+
def send_report(r, is_src = False):
s_failed = ' '.join([b.spec for b in r.batches if b.build_failed])
s_ok = ' '.join([b.spec for b in r.batches if not b.build_failed])
subject = 'upgrade failed '
subject += ' '.join((s_failed, s_ok)).strip()
-
+
m = mailer.Message()
m.set_headers(to = r.requester_email,
cc = config.builder_list,
for b in r.batches:
if b.build_failed and b.logfile == None:
info = b.skip_reason
- elif b.build_failed:
+ elif b.build_failed:
info = "FAILED"
- else:
+ else:
info = "OK"
m.write("%s (%s): %s\n" % (b.spec, b.branch, info))
m.write("\n\n*** buildlog for %s\n" % b.spec)
m.append_log(b.logfile)
m.write("\n\n")
-
+
m.send()
def send_cia_report(r, is_src = False):
subject = 'DeliverXML'
-
+
m = mailer.Message()
if (len(config.bot_email) == 0):
return
f = open(path.root_dir + '/PLD_Builder/cia-foot.xml')
m.write(f.read())
f.close()
-
+
# send the e-mail
m.send()
q.lock(0)
q.read()
for r in reqs:
- if r.kind != 'group':
+ if r.kind != 'group':
raise Exception, 'handle_reqs: fatal: huh? %s' % r.kind
need_it = 0
for b in r.batches:
sys.exit(1)
init_conf()
acl.try_reload()
-
+
status.push("fetching requests")
if has_new(config.control_url):
q = fetch_queue(config.control_url)
max_no = 0
q_new = []
for r in q:
- if r.no > max_no:
+ if r.no > max_no:
max_no = r.no
if r.no > last_count:
q_new.append(r)
f.close()
status.pop()
lck.close()
-
+
if __name__ == '__main__':
- # http connection is established (and few bytes transferred through it)
+ # http connection is established (and few bytes transferred through it)
# each $secs seconds.
loop.run_loop(main, secs = 10)
def check_double_id(id):
id_nl = id + "\n"
-
+
ids = open(path.processed_ids_file)
for i in ids.xreadlines():
if i == id_nl:
log.alert("request %s already processed" % id)
return 1
ids.close()
-
+
ids = open(path.processed_ids_file, "a")
ids.write(id_nl)
ids.close()
m.set_headers(subject = "building %s failed" % spec)
m.write_line(msg)
m.send()
-
+
lockf = lock("request")
if check_double_id(r.id):
lockf.close()
% (user.get_login(), pkg, bld, batch.branch))
lockf.close()
return
-
+
r.priority = user.check_priority(r.priority,config.builder)
r.requester = user.get_login()
r.requester_email = user.mail_to()
def ll(l):
util.append_to(b.logfile, l)
-
+
if b.files != []:
rpm_cache_dir = config.rpm_cache_dir
if "test-build" not in r.flags:
l.close()
else:
msg = "HIGH PRIORITY: "
-
+
msg += "handling request %s (%d) for %s from %s, priority %s" \
% (req.id, req.no, config.builder, req.requester, req.priority)
log.notice(msg)
if len(q.requests)<previouslen:
q.write()
q.unlock()
-
+
def main():
if len(sys.argv) < 2:
raise Exception, "fatal: need to have builder name as first arg"
return main_for(sys.argv[1])
-
+
if __name__ == '__main__':
loop.run_loop(main)
ret = q.requests[0]
q.requests = q.requests[1:]
return ret
-
+
def store_binary_request(r):
new_b = []
for b in r.batches:
else:
tag_test=" -Tp %s -tt" % (config.tag_prefixes[0],)
cmd = ("cd rpm/packages; nice -n %s ./builder %s -bs %s -r %s %s %s %s 2>&1" %
- (config.nice, builder_opts, b.bconds_string(), b.branch,
+ (config.nice, builder_opts, b.bconds_string(), b.branch,
tag_test, b.kernel_string(), b.spec))
util.append_to(b.logfile, "request from: %s" % r.requester)
util.append_to(b.logfile, "started at: %s" % time.asctime())
def format(self):
return "user:%.2fs sys:%.2fs real:%.2fs (faults io:%d non-io:%d)" % \
- (self.user_time, self.sys_time, self.time, self.io_faults,
+ (self.user_time, self.sys_time, self.time, self.io_faults,
self.non_io_faults)
-
+
class Timer:
def __init__(self):
self.starts = []
def pkg_name(nvr):
return re.match(r"(.+)-[^-]+-[^-]+", nvr).group(1)
-
+
def msg(m):
sys.stderr.write(m)
server = smtplib.SMTP(smtp_host)
# server.set_debuglevel(1)
-server.sendmail("matkor@pld-linux.org","builder-ac@pld-linux.org", msg) # Adresses should be taken from .requestrc
+server.sendmail("matkor@pld-linux.org","builder-ac@pld-linux.org", msg) # Adresses should be taken from .requestrc
server.quit()
projects / projects / pld-builder.new.git / commitdiff