- do stuff in loop, polling each 5 seconds so we get better response time

- bugfixes (especially deadlock gpg --clearsign and no locking in file_sender)

Changed files:
    PLD_Builder/file_sender.py -> 1.8
    PLD_Builder/gpg.py -> 1.7
    PLD_Builder/load_balancer.py -> 1.8
    PLD_Builder/loop.py -> 1.1
    PLD_Builder/pipeutil.py -> 1.1
    PLD_Builder/request.py -> 1.14
    PLD_Builder/request_fetcher.py -> 1.7
    PLD_Builder/rpm_builder.py -> 1.9
    PLD_Builder/srpm_builder.py -> 1.16
    PLD_Builder/wrap.py -> 1.3

diff --git a/PLD_Builder/file_sender.py b/PLD_Builder/file_sender.py
index 43675fabe861051c2e23fac0251531b3421a99ef..c96d4a268b01ec7bb3db023106be1593c25498a1 100644 (file)
--- a/PLD_Builder/file_sender.py
+++ b/PLD_Builder/file_sender.py
@@ -9,9 +9,10 @@ from acl import acl
 from config import config, init_conf
 import mailer
 import path
-import wrap
 import log
+import loop
 import status
+import lock
 
 retries_times = [5 * 60, 15 * 60, 60 * 60, 2 * 60 * 60, 5 * 60 * 60]
 
@@ -158,9 +159,11 @@ def flush_queue(dir):
 problem = ""
 
 def main():
+  if lock.lock("sending-files", non_block = 1) == None:
+    return
   init_conf("")
   maybe_flush_queue(path.buildlogs_queue_dir)
   maybe_flush_queue(path.ftp_queue_dir)
 
 if __name__ == '__main__':
-  wrap.wrap(main)
+  loop.run_loop(main)

diff --git a/PLD_Builder/gpg.py b/PLD_Builder/gpg.py
index 864c3460f0fb136bebbbaa95abb694f196bad4d6..acb79da40b585d156a3657a23d00e5792f6ad31a 100644 (file)
--- a/PLD_Builder/gpg.py
+++ b/PLD_Builder/gpg.py
@@ -3,6 +3,7 @@ import re
 import StringIO
 
 import util
+import pipeutil
 
 def verify_sig(buf):
   """Check signature.
@@ -12,30 +13,18 @@ def verify_sig(buf):
   object.
   """
   (gpg_out, gpg_in, gpg_err) = popen2.popen3("gpg --decrypt")
-  util.sendfile(buf, gpg_in)
-  gpg_in.close()
-  body = StringIO.StringIO()
-  for l in gpg_out.xreadlines():
-    body.write(l)
+  body = pipeutil.rw_pipe(buf, gpg_in, gpg_out)
   rx = re.compile("^gpg: Good signature from .*<([^>]+)>")
   emails = []
   for l in gpg_err.xreadlines():
     m = rx.match(l)
     if m:
       emails.append(m.group(1))
-  body.seek(0)
-  gpg_out.close()
   gpg_err.close()
   return (emails, body)
 
 def sign(buf):
   (gpg_out, gpg_in, gpg_err) = popen2.popen3("gpg --clearsign")
-  util.sendfile(buf, gpg_in)
-  gpg_in.close()
-  body = StringIO.StringIO()
-  for l in gpg_out.xreadlines():
-    body.write(l)
-  body.seek(0)
-  gpg_out.close()
+  body = pipeutil.rw_pipe(buf, gpg_in, gpg_out)
   gpg_err.close()
   return body

diff --git a/PLD_Builder/load_balancer.py b/PLD_Builder/load_balancer.py
index 881565113d7e00758652483f4d12878c5ba92fb6..87c7bc22beb03170cebc0b3dddddf400ec1f5340 100644 (file)
--- a/PLD_Builder/load_balancer.py
+++ b/PLD_Builder/load_balancer.py
@@ -7,7 +7,7 @@ import sys
 import log
 import status
 import lock
-import wrap
+import loop
 
 import rpm_builder
 
@@ -68,4 +68,4 @@ def main():
     pass
 
 if __name__ == '__main__':
-  wrap.wrap(main)
+  loop.run_loop(main)

diff --git a/PLD_Builder/loop.py b/PLD_Builder/loop.py
new file mode 100644 (file)
index 0000000..134b971
--- /dev/null
+++ b/PLD_Builder/loop.py
@@ -0,0 +1,29 @@
+import os
+import sys
+import time
+
+import wrap
+
+def run_loop(fnc, secs = 5, max = 60):
+  def run():
+    pid = os.fork()
+    if pid == 0:
+      wrap.wrap(fnc)
+      sys.exit(0)
+    else:
+      pid, s = os.waitpid(pid, 0)
+      if os.WIFEXITED(s):
+        s = os.WEXITSTATUS(s)
+        if s != 0:
+          sys.exit(s)
+      else:
+        sys.exit(10)
+      
+  start = time.time()
+  while time.time() - start < max:
+    last = time.time()
+    run()
+    took = time.time() - last
+    if took < secs:
+      time.sleep(secs - took)
+

diff --git a/PLD_Builder/pipeutil.py b/PLD_Builder/pipeutil.py
new file mode 100644 (file)
index 0000000..93c373c
--- /dev/null
+++ b/PLD_Builder/pipeutil.py
@@ -0,0 +1,34 @@
+import select
+import os
+import StringIO
+
+def rw_pipe(buf_, infd, outfd):
+  buf = StringIO.StringIO()
+  buf.write(buf_.read())
+  ret = StringIO.StringIO()
+  pos = 0
+  rd_fin = 0
+  wr_fin = 0
+  buf.seek(pos)
+  while not (rd_fin and wr_fin):
+    if wr_fin: o = []
+    else: o = [infd]
+    if rd_fin: i = []
+    else: i = [outfd]
+    i, o, e = select.select(i, o, [])
+    if i != []:
+      s = os.read(outfd.fileno(), 1000)
+      if s == "": rd_fin = 1
+      ret.write(s)
+    if o != []:
+      buf.seek(pos)
+      s = buf.read(1000)
+      if s == "":
+        infd.close()
+        wr_fin = 1
+      else:
+        cnt = os.write(infd.fileno(), s)
+        pos += cnt
+  outfd.close()
+  ret.seek(0)
+  return ret

diff --git a/PLD_Builder/request.py b/PLD_Builder/request.py
index 5ec002d326595ff2ba9903980c7b1ed1eb57f64a..e1cb8a523bf030bacb8efdd2b5dd69730d8bfcf8 100644 (file)
--- a/PLD_Builder/request.py
+++ b/PLD_Builder/request.py
@@ -31,8 +31,7 @@ def attr(e, a, default = None):
   except:
     if default != None:
       return default
-    # get the same exception
-    return e.attributes[a].value
+    raise
 
 def escape(s):
   return xml.sax.saxutils.escape(s)

diff --git a/PLD_Builder/request_fetcher.py b/PLD_Builder/request_fetcher.py
index 31a06dfec917a4d4205434660c350d115e84c673..e4e4ed673fcdf47cb1e1a8f24eaa5fb2e39011bd 100644 (file)
--- a/PLD_Builder/request_fetcher.py
+++ b/PLD_Builder/request_fetcher.py
@@ -12,7 +12,7 @@ import lock
 import util
 import gpg
 import request
-import wrap
+import loop
 from acl import acl
 from bqueue import B_Queue
 from config import config, init_conf
@@ -71,7 +71,7 @@ def handle_reqs(builder, reqs):
   q.unlock()
 
 def main():
-  lock.lock("request_fetcher")
+  lck = lock.lock("request_fetcher")
   init_conf("")
   
   status.push("fetching requests")
@@ -90,6 +90,9 @@ def main():
     f.write("%d\n" % max_no)
     f.close()
   status.pop()
+  lck.close()
   
 if __name__ == '__main__':
-  wrap.wrap(main)
+  # http connection is established (and few bytes transferred through it) 
+  # each $secs seconds.
+  loop.run_loop(main, secs = 10)

diff --git a/PLD_Builder/rpm_builder.py b/PLD_Builder/rpm_builder.py
index 4b76ee2f796b37c968298326f5e216bd9ea3cb45..5b86447a2498fb0fcec20619b9e54c1f865c3af6 100644 (file)
--- a/PLD_Builder/rpm_builder.py
+++ b/PLD_Builder/rpm_builder.py
@@ -9,7 +9,7 @@ from bqueue import B_Queue
 from acl import acl
 import lock
 import util
-import wrap
+import loop
 import path
 import status
 import log
@@ -105,12 +105,15 @@ def handle_request(r):
   notify.send()
 
 def check_load():
+  do_exit = 0
   try:
     f = open("/proc/loadavg")
     if float(string.split(f.readline())[2]) > config.max_load:
-      sys.exit(0)
+      do_exit = 1
   except:
     pass
+  if do_exit:
+    sys.exit(0)
 
 def main_for(builder):
   init_conf(builder)
@@ -160,4 +163,4 @@ def main():
   return main_for(sys.argv[1])
   
 if __name__ == '__main__':
-  wrap.wrap(main)
+  loop.run_loop(main)

diff --git a/PLD_Builder/srpm_builder.py b/PLD_Builder/srpm_builder.py
index bfe863593be7034cdff233235e4fd00121b4d952..94c438f1c7aec5981c126727e96c140f6d98a6d3 100644 (file)
--- a/PLD_Builder/srpm_builder.py
+++ b/PLD_Builder/srpm_builder.py
@@ -13,7 +13,7 @@ import request
 import log
 import path
 import util
-import wrap
+import loop
 import chroot
 import buildlogs
 import ftp
@@ -131,4 +131,4 @@ def main():
   status.pop()
 
 if __name__ == '__main__':
-  wrap.wrap(main)
+  loop.run_loop(main)

diff --git a/PLD_Builder/wrap.py b/PLD_Builder/wrap.py
index 96166019388f8cea60cceceaf28fa24808efd633..d8f77d26b9ea5cbe640181b0dad15b9df622517c 100644 (file)
--- a/PLD_Builder/wrap.py
+++ b/PLD_Builder/wrap.py
@@ -3,6 +3,7 @@ import log
 import traceback
 import StringIO
 import os
+import time
 
 # this module, as it deals with internal error handling shouldn't
 # import anything beside status