sio.seek(0)
(fdno, tmpname) = tempfile.mkstemp(dir=os.path.dirname(name))
f = os.fdopen(fdno, "w")
- util.sendfile(sio, f)
+ shutil.copyfileobj(sio, f)
f.flush()
os.fsync(f.fileno())
f.close()
import re
import random
import util
-
-try:
- from hashlib import md5 as md5
-except ImportError:
- from md5 import md5
+import shutil
+import subprocess
from config import config
return "%s sudo chroot %s /bin/sh -c \"export LC_ALL=C; exec < /dev/null; %s\"" \
% (config.sudo_chroot_wrapper, config.chroot, quote(cmd))
-def popen(cmd, user = "builder", mode = "r"):
- f = os.popen(command(cmd, user), mode)
+def popen(cmd, user = "builder", mode = "r", encoding = None):
+ if mode == "r":
+ p = subprocess.Popen(command(cmd, user), shell=True, stdout=subprocess.PIPE, close_fds=True, encoding=encoding)
+ f = p.stdout
+ else:
+ p = subprocess.Popen(command(cmd, user), shell=True, stdin=subprocess.PIPE, close_fds=True, encoding=encoding)
+ f = p.stdin
return f
def run(cmd, user = "builder", logfile = None, logstdout = None):
return r
def cp(file, outfile, user="builder", rm=False):
- m = md5()
- m.update(util.to_bytes(str(random.sample(range(100000), 500))))
- digest = m.hexdigest()
-
- marker_start = "--- FILE BEGIN DIGEST %s ---" % digest
- marker_end = "--- FILE END DIGEST %s ---" % digest
-
- f = open(outfile, 'wb')
- cmd = "echo \"%s\"; cat %s; echo \"%s\"" % (marker_start, file, marker_end)
+ f_out = open(outfile, 'wb')
+ cmd = "cat %s" % file
if rm:
cmd += "; rm %s" % file
- c = command(cmd, user)
- p = os.popen(c)
- # get file contents
- marker = False
- for l in p:
- if not marker and l.strip() == marker_start:
- marker = True
- continue
- me = l.find(marker_end)
- if me != -1:
- l = l[:me]
- f.write(util.to_bytes(l))
- marker = False
- break
- if marker:
- f.write(util.to_bytes(l))
- rp = p.close()
- rf = f.close()
- if rp == None:
+ p = subprocess.Popen(command(cmd, user), shell=True, stdout=subprocess.PIPE, close_fds=True)
+ f_in = p.stdout
+ shutil.copyfileobj(f_in, f_out)
+ f_out.close()
+ r = f_in.close()
+ if r == None:
return 0
else:
- return rp
+ return r
import shutil
import sys
import traceback
-import urllib2
-
+import urllib.request
from config import config, init_conf
import mailer
import path
global problems
try:
f = open(src, 'r')
- data = f.read()
+ data = f.read().encode('utf-8')
f.close()
- req = urllib2.Request(url, data)
- req.add_header('X-Filename', os.path.basename(src))
- f = urllib2.urlopen(req)
+ headers = { 'X-Filename' : os.path.basename(src) }
+ req = urllib.request.Request(url, data=data, headers=headers)
+ f = urllib.request.urlopen(req)
f.close()
except Exception as e:
problems[src] = e
d = read_name_val(f)
if d != None: q.append(d)
def mycmp(x, y):
- rc = cmp(x['Time'], y['Time'])
+ rc = util.cmp(x['Time'], y['Time'])
if rc == 0 and 'Type' in x and 'Type' in y:
- return cmp(x['Type'], y['Type'])
+ return util.cmp(x['Type'], y['Type'])
else:
return rc
q.sort(key=util.cmp_to_key(mycmp))
del killset[p]
errors += "cannot remove %s because it's crucial\n" % p
else:
- f = chroot.popen("poldek --noask --test --test --erase %s" % p, user = "root")
+ f = chroot.popen("poldek --noask --test --test --erase %s" % p, user = "root", encoding = "utf-8")
crucial = 0
e = []
for l in f:
return errors
def upgrade_from_batch(r, b):
- f = chroot.popen("rpm --test -F %s 2>&1" % ' '.join(b.files), user = "root")
+ f = chroot.popen("rpm --test -F %s 2>&1" % ' '.join(b.files), user = "root", encoding = "utf-8")
killset = {}
rx = re.compile(r' \(installed\) (?P<name>[^\s]+)-[^-]+-[^-]+$')
for l in f:
'rpmdefs' : b.rpmbuild_opts(),
'topdir' : b.get_topdir(),
'spec': b.spec,
- })
+ }, encoding = "utf-8")
# java-sun >= 1.5 conflicts with soprano-2.1.67-1.src
# java-sun conflicts with soprano-2.1.67-1.src
rx = re.compile(r"\s+(?P<name>[\w-]+)\s+.*conflicts with [^\s]+-[^-]+-[^-]+\.src($| .*)")
def install_br(r, b):
def is_rpmorg():
- f = chroot.popen("rpm --version 2>&1")
+ f = chroot.popen("rpm --version 2>&1", encoding = "utf-8")
v = re.compile(r'(RPM version|rpm \(RPM\)) (?P<major>\d)\.(?P<minor>\d+)(\.\d+)?')
for l in f:
m = v.search(l)
'rpmdefs' : b.rpmbuild_opts(),
'spec': b.spec,
}
- f = chroot.popen(cmd)
+ f = chroot.popen(cmd, encoding = "utf-8")
rx = re.compile(r"^\s*(?P<name>[^\s]+) .*is needed by")
needed = {}
b.log_line("checking BR")
chroot.run("poldek --up --upa", user = "root", logfile = b.logfile)
# check conflicts in BRed packages
b.log_line("checking conflicting packages in BRed packages")
- f = chroot.popen("poldek --test --test --noask --caplookup -Q -v %s --upgrade %s" % (b.ignores(), br), user = "root")
+ f = chroot.popen("poldek --test --test --noask --caplookup -Q -v %s --upgrade %s" % (b.ignores(), br), user = "root", encoding = "utf-8")
# phonon-devel-4.3.1-1.i686 conflicts with qt4-phonon-devel-4.5.0-6.i686
# jdbc-stdext >= 2.0 is required by installed java-struts-1.3.10-1.noarch
# jmx is needed by (installed) java-commons-modeler-2.0-1.noarch
log.alert("found strange lock in got-lock: %s" % b)
def mycmp(b1, b2):
- return cmp(bs[b1], bs[b2])
+ return util.cmp(bs[b1], bs[b2])
bl.sort(key=util.cmp_to_key(mycmp))
import sys
from io import StringIO
from config import config
-import util
+import shutil
import log
def recode(s):
self.body.write("\n\n[...]\n\n")
line += 1
else:
- util.sendfile(open(log), self.body)
+ shutil.copyfileobj(open(log), self.body)
def set_std_headers(self):
self.headers["Date"] = time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime())
f.write("%s: %s\n" % (k, v))
f.write("\n")
self.body.seek(0)
- util.sendfile(self.body, f)
+ shutil.copyfileobj(self.body, f)
def send(self):
if not os.path.exists("/usr/lib/sendmail"):
def handle_bin():
send_rpmqa()
- f=chroot.popen("""ls -l --time-style +%s /spools/ready""", 'root')
+ f=chroot.popen("""ls -l --time-style +%s /spools/ready""", "root", encoding = "utf-8")
rmpkgs=[]
curtime=time.time()
for i in f:
import xml.sax.saxutils
import fnmatch
import os
-import urllib
+import urllib.parse
import html
import pytz
import tempfile
import string
import signal
import os
-import urllib
-import urllib2
+import urllib.request
import sys
-from io import StringIO
+from io import StringIO, BytesIO
import gzip
import path
import log
import status
import lock
import util
+import shutil
import gpg
import request
import loop
signal.alarm(300)
try:
headers = { 'Cache-Control': 'no-cache', 'Pragma': 'no-cache' }
- req = urllib2.Request(url=control_url + "/max_req_no", headers=headers)
- f = urllib2.urlopen(req)
+ req = urllib.request.Request(url=control_url + "/max_req_no", headers=headers)
+ f = urllib.request.urlopen(req)
count = int(f.readline().strip())
signal.alarm(0)
except Exception as e:
signal.alarm(300)
try:
headers = { 'Cache-Control': 'no-cache', 'Pragma': 'no-cache' }
- req = urllib2.Request(url=control_url + "/queue.gz", headers=headers)
- f = urllib2.urlopen(req)
+ req = urllib.request.Request(url=control_url + "/queue.gz", headers=headers)
+ f = urllib.request.urlopen(req)
signal.alarm(0)
except Exception as e:
signal.alarm(0)
log.error("can't fetch %s: %s" % (control_url + "/queue.gz", e))
sys.exit(1)
- sio = StringIO()
- util.sendfile(f, sio)
+ sio = BytesIO()
+ shutil.copyfileobj(f, sio)
f.close()
sio.seek(0)
f = gzip.GzipFile(fileobj = sio)
from bqueue import B_Queue
from config import config, init_conf
from mailer import Message
-#import messagebus
def check_double_id(id):
id_nl = id + "\n"
status.push("request from %s" % user.login)
r = request.parse_request(body)
if r.kind == 'group':
-# messagebus.notify(topic="request.group", user=user.login, **r.dump_json())
handle_group(r, user)
elif r.kind == 'notification':
-# messagebus.notify(topic="request.notify", user=user.login, **r.dump_json())
handle_notification(r, user)
else:
msg = "%s: don't know how to handle requests of this kind '%s'" \
import time
import datetime
import string
-import urllib
-import urllib2
+import urllib.request
+import urllib.parse
+import urllib.error
from config import config, init_conf
from bqueue import B_Queue
import lock
import util
+import shutil
import loop
import path
import status
def mycmp(r1, r2):
if r1.kind != 'group' or r2.kind != 'group':
raise Exception("non-group requests")
- pri_diff = cmp(r1.priority, r2.priority)
+ pri_diff = util.cmp(r1.priority, r2.priority)
if pri_diff == 0:
- return cmp(r1.time, r2.time)
+ return util.cmp(r1.time, r2.time)
else:
return pri_diff
q.requests.sort(key=util.cmp_to_key(mycmp))
while not good:
try:
headers = { 'Cache-Control': 'no-cache', 'Pragma': 'no-cache' }
- req = urllib2.Request(url=src_url, headers=headers)
- f = urllib2.urlopen(req)
+ req = urllib.request.Request(url=src_url, headers=headers)
+ f = urllib.request.urlopen(req)
good = True
- except urllib2.HTTPError as error:
+ except urllib.error.HTTPError as error:
return False
- except urllib2.URLError as error:
+ except urllib.error.URLError as error:
# see errno.h
try:
errno = error.errno
while not good:
try:
headers = { 'Cache-Control': 'no-cache', 'Pragma': 'no-cache' }
- req = urllib2.Request(url=src_url, headers=headers)
- f = urllib2.urlopen(req)
+ req = urllib.request.Request(url=src_url, headers=headers)
+ f = urllib.request.urlopen(req)
good = True
- except urllib2.HTTPError as error:
+ except urllib.error.HTTPError as error:
# fail in a way where cron job will retry
msg = "unable to fetch url %s, http code: %d" % (src_url, error.code)
b.log_line(msg)
msg = "in queue for more than 6 hours, download failing"
b.log_line(msg)
return False
- except urllib2.URLError as error:
+ except urllib.error.URLError as error:
errno = 0
if isinstance(error.args[0], IOError):
errno = error.args[0].errno
print("error.reason exception %s" % e)
raise
- o = chroot.popen("cat > %s" % b.src_rpm, mode = "wb")
+ o = chroot.popen("cat > %s" % b.src_rpm, mode = "w")
try:
- bytes = util.sendfile(f, o)
+ shutil.copyfileobj(f, o)
except IOError as e:
b.log_line("error: unable to write to `%s': %s" % (b.src_rpm, e))
raise
+ bytes = float(f.headers['content-length'])
f.close()
o.close()
t = time.time() - start
import status
import build
import report
-#import messagebus
from lock import lock
from bqueue import B_Queue
def mycmp(r1, r2):
if r1.kind != 'group' or r2.kind != 'group':
raise Exception("non-group requests")
- pri_diff = cmp(r1.priority, r2.priority)
+ pri_diff = util.cmp(r1.priority, r2.priority)
if pri_diff == 0:
- return cmp(r1.time, r2.time)
+ return util.cmp(r1.time, r2.time)
else:
return pri_diff
q.requests.sort(key=util.cmp_to_key(mycmp))
return "FAIL"
status.push("building %s" % b.spec)
-# messagebus.notify(topic="build_srpm.start", spec=b.spec, flags=r.flags, batch=b, request=r)
b.src_rpm = ""
builder_opts = "-nu -nm --nodeps --http --define \'_pld_builder 1\'"
if res:
res = "FAIL"
-# messagebus.notify(topic="build_srpm.finish", spec=b.spec)
return res
def handle_request(r):
def msg(m):
sys.stderr.write(m)
-def sendfile(src, dst):
- cnt = 0
- while 1:
- s = src.read(10000)
- if s == "": break
- cnt += len(s)
- dst.write(s)
- return cnt
-
def append_to(log, msg):
f = open(log, "a")
f.write("%s\n" % msg)
return s
else:
raise TypeError("Expected bytes or string, but got %s." % type(s))
+
+def cmp(a, b):
+ return (a > b) - (a < b)
projects / projects / pld-builder.new.git / commitdiff