- allow logging via syslog

[projects/pld-builder.new.git] / PLD_Builder / rpm_builder.py

import sys
import os
import atexit
import time
import urllib

from config import config, init_conf
from bqueue import B_Queue
from acl import acl
import lock
import util
import loop
import path
import status
import log
import chroot
import ftp
import notify
import build
import report

# this code is duplicated in srpm_builder, but we
# might want to handle some cases differently here
def pick_request(q):
  def mycmp(r1, r2):
    if r1.kind != 'group' or r2.kind != 'group':
      raise "non-group requests"
    pri_diff = cmp(r1.priority, r2.priority)
    if pri_diff == 0:
      return cmp(r1.time, r2.time)
    else:
      return pri_diff
  q.requests.sort(mycmp)
  ret = q.requests[0]
  q.requests = q.requests[1:]
  return ret

def fetch_src(r, b):
  src_url = config.control_url + "/srpms/" + r.id + "/" + b.src_rpm
  b.log_line("fetching %s" % src_url)
  start = time.time()
  f = urllib.urlopen(src_url)
  o = chroot.popen("cat > %s" % b.src_rpm, mode = "w")
  bytes = util.sendfile(f, o)
  f.close()
  o.close()
  t = time.time() - start
  if t == 0:
    b.log_line("fetched %d bytes" % bytes)
  else:
    b.log_line("fetched %d bytes, %.1f K/s" % (bytes, bytes / 1024.0 / t))

def build_rpm(r, b):
  status.push("building %s" % b.spec)
  b.log_line("request from: %s" % r.requester)
  b.log_line("started at: %s" % time.asctime())
  fetch_src(r, b)
  b.log_line("installing srpm: %s" % b.src_rpm)
  res = chroot.run("rpm -U %s" % b.src_rpm, logfile = b.logfile)
  chroot.run("rm -f %s" % b.src_rpm, logfile = b.logfile)
  if res:
    b.log_line("error: installing src rpm failed")
    res = 1
  else:
    cmd = "install -m 700 -d $HOME/%s; cd rpm/SPECS; " \
          "TMPDIR=$HOME/%s rpmbuild -bb %s" % \
          (b.b_id, b.b_id, b.spec)
    b.log_line("building RPM using: %s" % cmd)
    res = chroot.run(cmd, logfile = b.logfile)
    files = util.collect_files(b.logfile)
    if len(files) > 0:
      r.chroot_files.extend(files)
    else:
      # FIXME: is it error?
      b.log_line("error: No files produced.")
      res = 1
    b.files = files
  chroot.run("rm -rf $HOME/%s; cd rpm/SPECS; rpmbuild --nodeps --nobuild " \
             "--clean --rmspec --rmsource %s" % \
             (b.b_id, b.spec), logfile = b.logfile)
  
  for f in b.files:
    local = r.tmp_dir + os.path.basename(f)
    chroot.run("cat %s; rm -f %s" % (f, f), logfile = local)
    ftp.add(local)
        
  status.pop()

  return res

def handle_request(r):
  ftp.init(r)
  build.build_all(r, build_rpm)
  report.send_report(r)
  ftp.flush()
  notify.send()

def check_load():
  do_exit = 0
  try:
    f = open("/proc/loadavg")
    if float(string.split(f.readline())[2]) > config.max_load:
      do_exit = 1
  except:
    pass
  if do_exit:
    sys.exit(0)

def main_for(builder):
  init_conf(builder)
  # allow only one build in given builder at once
  if not lock.lock("building-rpm-for-%s" % config.builder, non_block = 1):
    return
  # don't kill server
  check_load()
  # not more then job_slots builds at once
  locked = 0
  for slot in range(config.job_slots):
    if lock.lock("building-rpm-slot-%d" % slot, non_block = 1):
      locked = 1
      break
  if not locked:
    return

  status.push("picking request for %s" % config.builder)
  q = B_Queue(path.queue_file + "-" + config.builder)
  q.lock(0)
  q.read()
  if q.requests == []:
    return
  r = pick_request(q)
  q.write()
  q.unlock()
  status.pop()

  # record fact that we got lock for this builder, load balancer
  # will use it for fair-queuing
  l = lock.lock("got-lock")
  f = open(path.got_lock_file, "a")
  f.write(config.builder + "\n")
  f.close()
  l.close()
  
  msg = "handling request %s (%d) for %s from %s" \
        % (r.id, r.no, config.builder, r.requester)
  log.notice(msg)
  status.push(msg)
  handle_request(r)
  status.pop()
  
def main():
  if len(sys.argv) < 2:
    raise "fatal: need to have builder name as first arg"
  return main_for(sys.argv[1])
  
if __name__ == '__main__':
  loop.run_loop(main)