- load balancer done

- some cleanups

Changed files:
    PLD_Builder/load_balancer.py -> 1.1
    PLD_Builder/request_fetcher.py -> 1.3
    admin/fresh-queue.sh -> 1.7
    bin/load-balancer.sh -> 1.1
    config/builder.conf -> 1.5
    doc/README -> 1.4
    doc/TODO -> 1.5

diff --git a/PLD_Builder/load_balancer.py b/PLD_Builder/load_balancer.py
new file mode 100644 (file)
index 0000000..2904498
--- /dev/null
+++ b/PLD_Builder/load_balancer.py
@@ -0,0 +1,60 @@
+import string
+import os
+import time
+
+import path
+import log
+import status
+import lock
+import util
+from config import config, init_conf
+
+# return list of binary builders in fair-queue order
+# it is determined based upon spool/got_lock file, which is also
+# updated to be short
+def builders_order():
+  bs = {}
+  bl = []
+  for b in config.builders:
+    bs[b] = 0
+    bl.append(b)
+    
+  lck = lock.lock("got-lock")
+  f = open(path.got_lock_file, "r+")
+  line_no = 0
+  
+  for l in f.xreadlines():
+    line_no += 1
+    b = string.strip(l)
+    if bs.has_key(b):
+      bs[b] = line_no
+    else:
+      log.alert("found strange lock in got-lock: %s" % b)
+
+  bl.sort(lambda (b1, b2): cmp(bs[b1], bs[b2]))
+
+  f.seek(0)
+  f.truncate(0)
+  for l in bl: f.write(l + "\n")
+  f.close()
+  lck.close()
+
+  return bl
+
+def run_rpm_builder(b):
+  prog = path.root_dir + "binary_builder/rpm-builder.sh"
+  os.spawnl(os.P_NOWAIT, prog, prog, b)
+
+def main():
+  init_conf("")
+  for b in builders_order():
+    run_rpm_builder(b)
+    # give builder some time to aquire lock
+    time.sleep(1)
+  # wait for children to die out
+  try:
+    while 1: os.wait()
+  except:
+    pass
+
+util.wrap(main)

diff --git a/PLD_Builder/request_fetcher.py b/PLD_Builder/request_fetcher.py
index ac915ca73b8bb22f96cbc0dfa98591b4bc4b5a57..6990ab3a7bba4ea95029baf568b47d009e1ec89c 100644 (file)
--- a/PLD_Builder/request_fetcher.py
+++ b/PLD_Builder/request_fetcher.py
@@ -1,4 +1,3 @@
-import ConfigParser
 import string
 import os
 import urllib

diff --git a/admin/fresh-queue.sh b/admin/fresh-queue.sh
index 5a876b7dc47675e56b30948941074096af5d3984..04d842228d64266c68eacd4ce9e34b2c60c3a1f6 100755 (executable)
--- a/admin/fresh-queue.sh
+++ b/admin/fresh-queue.sh
@@ -10,5 +10,6 @@ mkdir -p spool/{buildlogs,ftp} www/srpms log lock
 echo 0 > www/max_req_no
 echo 0 > spool/last_req_no
 echo -n > spool/processed_ids
+echo -n > spool/got_lock
 echo '<queue/>' > spool/queue
 echo '<queue/>' > spool/req_queue

diff --git a/bin/load-balancer.sh b/bin/load-balancer.sh
new file mode 100644 (file)
index 0000000..f85d569
--- /dev/null
+++ b/bin/load-balancer.sh
@@ -0,0 +1,4 @@
+#!/bin/sh
+
+cd ~/pld-builder.new
+python PLD_Builder/load_balancer.py

diff --git a/config/builder.conf b/config/builder.conf
index 16e05ab8b9c662a40ec8af575624d3836fc009b1..744da129e24213ee25e6cd31b1ff7736638fa786 100644 (file)
--- a/config/builder.conf
+++ b/config/builder.conf
@@ -7,8 +7,8 @@ job_slots = 1
 max_load = 10.0
 # where to look for queue.gz and counter
 control_url = http://roke.freak/builder/
-# list of all builders
-builders = src athlon
+# list of binary builders
+builders = athlon
 # login of user (from acl.conf) that should sign queue
 queue_signed_by = malekith
 

diff --git a/doc/README b/doc/README
index ee6c13c4ed519a1f033529afecbcce12f6e57d27..0cc189e03cb8a32cba98837927d9f127e86af932 100644 (file)
--- a/doc/README
+++ b/doc/README
@@ -35,6 +35,12 @@ Roadmap:
     rpm-builder.sh <builder>
       Tries to aquire locks for <builder> and for job-slot. If that suceeds, 
       proccess one request from spool/queue-<builder>.
+
+    load-balancer.sh
+      Checks spool/got_lock. Then run rpm-builder.sh in order determined
+      from this file (if b1 had lock last time before b2, first run
+      rpm-builder.sh b1 and then rpm-builder.sh b2), so builders get
+      the same number of requests.
   
 Working directories:
   lock/

diff --git a/doc/TODO b/doc/TODO
index 2e63dfa180e193c5110a48cd65c5f2159bb475a5..88b908ab4a29c8d6060ee27e0790d11c4dbb633c 100644 (file)
--- a/doc/TODO
+++ b/doc/TODO
@@ -1,9 +1,6 @@
 - file-sender -- buildlogs, ftp queues. support for file://, scp://, rsync://
   remember to send files in order, they were stored
 
-- load-balancer -- process got_lock, and run rpm-builder for archs in proper
-  order, shorten got_lock
-  
 - make mailer.py actually send mail (easy)
 
 - fix make-request.sh to be more user-friendly and configurable