PLD_Builder/request.py

   1 # vi: encoding=utf-8 ts=8 sts=4 sw=4 et
   2
   3 from xml.dom.minidom import *
   4 import string
   5 import time
   6 import xml.sax.saxutils
   7 import fnmatch
   8 import os
   9 import urllib
  10 import cgi
  11
  12 import util
  13 import log
  14 from acl import acl
  15 from config import config
  16
  17 __all__ = ['parse_request', 'parse_requests']
  18
  19 def text(e):
  20     res = ""
  21     for n in e.childNodes:
  22         if n.nodeType != Element.TEXT_NODE:
  23             log.panic("xml: text expected in <%s>, got %d" % (e.nodeName, n.nodeType))
  24         res += n.nodeValue
  25     return res
  26
  27 def attr(e, a, default = None):
  28     try:
  29         return e.attributes[a].value
  30     except:
  31         if default != None:
  32             return default
  33         raise
  34
  35 def escape(s):
  36     return xml.sax.saxutils.escape(s)
  37
  38 # return timestamp with timezone information
  39 # so we could parse it in javascript
  40 def tzdate(t):
  41     # as strftime %z is unofficial, and does not work, need to make it numeric ourselves
  42 #    date = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(t))
  43     date = time.strftime("%a %b %d %Y %H:%M:%S", time.localtime(t))
  44     # NOTE: the altzone is showing CURRENT timezone, not what the "t" reflects
  45     # NOTE: when DST is off timezone gets it right, altzone not
  46     if time.daylight:
  47         tzoffset = time.altzone
  48     else:
  49         tzoffset = time.timezone
  50     tz = '%+05d' % (-tzoffset / 3600 * 100)
  51     return date + ' ' + tz
  52
  53 def is_blank(e):
  54     return e.nodeType == Element.TEXT_NODE and string.strip(e.nodeValue) == ""
  55
  56 class Group:
  57     def __init__(self, e):
  58         self.batches = []
  59         self.kind = 'group'
  60         self.id = attr(e, "id")
  61         self.no = int(attr(e, "no"))
  62         self.priority = 2
  63         self.time = time.time()
  64         self.requester = ""
  65         self.max_jobs = 0
  66         self.requester_email = ""
  67         self.flags = string.split(attr(e, "flags", ""))
  68         for c in e.childNodes:
  69             if is_blank(c): continue
  70
  71             if c.nodeType != Element.ELEMENT_NODE:
  72                 log.panic("xml: evil group child %d" % c.nodeType)
  73             if c.nodeName == "batch":
  74                 self.batches.append(Batch(c))
  75             elif c.nodeName == "requester":
  76                 self.requester = text(c)
  77                 self.requester_email = attr(c, "email", "")
  78             elif c.nodeName == "priority":
  79                 self.priority = int(text(c))
  80             elif c.nodeName == "time":
  81                 self.time = int(text(c))
  82             elif c.nodeName == "maxjobs":
  83                 self.max_jobs = int(text(c))
  84             else:
  85                 log.panic("xml: evil group child (%s)" % c.nodeName)
  86         # note that we also check that group is sorted WRT deps
  87         m = {}
  88         for b in self.batches:
  89             deps = []
  90             m[b.b_id] = b
  91             for dep in b.depends_on:
  92                 if m.has_key(dep):
  93                     # avoid self-deps
  94                     if id(m[dep]) != id(b):
  95                         deps.append(m[dep])
  96                 else:
  97                     log.panic("xml: dependency not found in group")
  98             b.depends_on = deps
  99         if self.requester_email == "" and self.requester != "":
 100             self.requester_email = acl.user(self.requester).mail_to()
 101
 102     def dump(self, f):
 103         f.write("group: %d (id=%s pri=%d)\n" % (self.no, self.id, self.priority))
 104         f.write("  from: %s\n" % self.requester)
 105         f.write("  flags: %s\n" % string.join(self.flags))
 106         f.write("  time: %s\n" % time.asctime(time.localtime(self.time)))
 107         for b in self.batches:
 108             b.dump(f)
 109         f.write("\n")
 110
 111     def dump_html(self, f):
 112         f.write(
 113             "<div id=\"%(no)d\" class=\"%(flags)s\">\n"
 114             "<a href=\"#%(no)d\">%(no)d</a>. <span id=\"tz\">%(time)s</span> from <b>%(requester)s</b> "
 115             "<small>%(id)s, prio=%(priority)d, jobs=%(max_jobs)d, %(flags)s</small>\n"
 116         % {
 117             'no': self.no,
 118             'id': '<a href="srpms/%(id)s">%(id)s</a>' % {'id': self.id},
 119             'time': escape(tzdate(self.time)),
 120             'requester': escape(self.requester),
 121             'priority': self.priority,
 122             'max_jobs': self.max_jobs,
 123             'flags': string.join(self.flags)
 124         })
 125         f.write("<ul>\n")
 126         for b in self.batches:
 127             b.dump_html(f, self.id)
 128         f.write("</ul>\n")
 129         f.write("</div>\n")
 130
 131     def write_to(self, f):
 132         f.write("""
 133        <group id="%s" no="%d" flags="%s">
 134          <requester email='%s'>%s</requester>
 135          <time>%d</time>
 136          <priority>%d</priority>
 137          <maxjobs>%d</maxjobs>\n""" % (self.id, self.no, string.join(self.flags),
 138                     escape(self.requester_email), escape(self.requester),
 139                     self.time, self.priority, self.max_jobs))
 140         for b in self.batches:
 141             b.write_to(f)
 142         f.write("       </group>\n\n")
 143
 144     def is_done(self):
 145         ok = 1
 146         for b in self.batches:
 147             if not b.is_done():
 148                 ok = 0
 149         return ok
 150
 151 class Batch:
 152     def __init__(self, e):
 153         self.bconds_with = []
 154         self.bconds_without = []
 155         self.builders = []
 156         self.builders_status = {}
 157         self.builders_status_time = {}
 158         self.builders_status_buildtime = {}
 159         self.kernel = ""
 160         self.defines = {}
 161         self.target = []
 162         self.branch = ""
 163         self.src_rpm = ""
 164         self.info = ""
 165         self.spec = ""
 166         self.command = ""
 167         self.command_flags = []
 168         self.skip = []
 169         self.gb_id = ""
 170         self.b_id = attr(e, "id")
 171         self.depends_on = string.split(attr(e, "depends-on"))
 172         self.upgraded = True
 173         for c in e.childNodes:
 174             if is_blank(c): continue
 175
 176             if c.nodeType != Element.ELEMENT_NODE:
 177                 log.panic("xml: evil batch child %d" % c.nodeType)
 178             if c.nodeName == "src-rpm":
 179                 self.src_rpm = text(c)
 180             elif c.nodeName == "spec":
 181                 # normalize specname, specname is used as buildlog and we don't
 182                 # want to be exposed to directory traversal attacks
 183                 self.spec = text(c).split('/')[-1]
 184             elif c.nodeName == "command":
 185                 self.spec = "COMMAND"
 186                 self.command = text(c).strip()
 187                 self.command_flags = string.split(attr(c, "flags", ""))
 188             elif c.nodeName == "info":
 189                 self.info = text(c)
 190             elif c.nodeName == "kernel":
 191                 self.kernel = text(c)
 192             elif c.nodeName == "define":
 193                 define = attr(c, "name")
 194                 self.defines[define] = text(c)
 195             elif c.nodeName == "target":
 196                 self.target.append(text(c))
 197             elif c.nodeName == "skip":
 198                 self.skip.append(text(c))
 199             elif c.nodeName == "branch":
 200                 self.branch = text(c)
 201             elif c.nodeName == "builder":
 202                 key = text(c)
 203                 self.builders.append(key)
 204                 self.builders_status[key] = attr(c, "status", "?")
 205                 self.builders_status_time[key] = attr(c, "time", "0")
 206                 self.builders_status_buildtime[key] = "0" #attr(c, "buildtime", "0")
 207             elif c.nodeName == "with":
 208                 self.bconds_with.append(text(c))
 209             elif c.nodeName == "without":
 210                 self.bconds_without.append(text(c))
 211             else:
 212                 log.panic("xml: evil batch child (%s)" % c.nodeName)
 213
 214     def is_done(self):
 215         ok = 1
 216         for b in self.builders:
 217             s = self.builders_status[b]
 218             if not s.startswith("OK") and not s.startswith("SKIP") and not s.startswith("UNSUPP") and not s.startswith("FAIL"):
 219                 ok = 0
 220         return ok
 221
 222     def dump(self, f):
 223         f.write("  batch: %s/%s\n" % (self.src_rpm, self.spec))
 224         f.write("    info: %s\n" % self.info)
 225         f.write("    kernel: %s\n" % self.kernel)
 226         f.write("    defines: %s\n" % self.defines_string())
 227         f.write("    target: %s\n" % self.target_string())
 228         f.write("    branch: %s\n" % self.branch)
 229         f.write("    bconds: %s\n" % self.bconds_string())
 230         builders = []
 231         for b in self.builders:
 232             builders.append("%s:%s" % (b, self.builders_status[b]))
 233         f.write("    builders: %s\n" % string.join(builders))
 234
 235     def is_command(self):
 236         return self.command != ""
 237
 238     def dump_html(self, f, rid):
 239         f.write("<li>\n")
 240         if self.is_command():
 241             desc = "SH: <pre>%s</pre> flags: [%s]" % (self.command, ' '.join(self.command_flags))
 242         else:
 243             package_url = "http://git.pld-linux.org/gitweb.cgi?p=packages/%(package)s.git;f=%(spec)s;h=%(branch)s;a=shortlog" % {
 244                 'spec': self.spec,
 245                 'branch': self.branch,
 246                 'package': self.spec[:-5],
 247             }
 248             desc = "%(src_rpm)s (<a href=\"%(package_url)s\">%(spec)s -r %(branch)s</a>%(rpmopts)s)" % {
 249                 'src_rpm': self.src_rpm,
 250                 'spec': self.spec,
 251                 'branch': self.branch,
 252                 'rpmopts': self.bconds_string() + self.kernel_string() + self.target_string() + self.defines_string(),
 253                 'package_url': package_url,
 254             }
 255         f.write("%s <small>[" % desc)
 256         builders = []
 257         for b in self.builders:
 258             s = self.builders_status[b]
 259             if s.startswith("OK"):
 260                 c = "green"
 261             elif s.startswith("FAIL"):
 262                 c = "red"
 263             elif s.startswith("SKIP"):
 264                 c = "blue"
 265             elif s.startswith("UNSUPP"):
 266                 c = "fuchsia"
 267             else:
 268                 c = "black"
 269             link_pre = ""
 270             link_post = ""
 271             if (s.startswith("OK") or s.startswith("SKIP") or s.startswith("UNSUPP") or s.startswith("FAIL")) and len(self.spec) > 5:
 272                 if self.is_command():
 273                     bl_name = "command"
 274                 else:
 275                     bl_name = self.spec[:len(self.spec)-5]
 276                 lin_ar = b.replace('noauto-','')
 277                 path = "/%s/%s/%s,%s.bz2" % (lin_ar.replace('-','/'), s, bl_name, rid)
 278                 is_ok = 0
 279                 if s.startswith("OK"):
 280                     is_ok = 1
 281                 bld = lin_ar.split('-')
 282                 tree_name = '-'.join(bld[:-1])
 283                 tree_arch = '-'.join(bld[-1:])
 284                 link_pre = "<a href=\"http://buildlogs.pld-linux.org/index.php?dist=%s&arch=%s&ok=%d&name=%s&id=%s&action=tail\">" \
 285                         % (urllib.quote(tree_name), urllib.quote(tree_arch), is_ok, urllib.quote(bl_name), urllib.quote(rid))
 286                 link_post = "</a>"
 287
 288             def ftime(s):
 289                 t = float(s)
 290                 if t > 0:
 291                     return time.asctime(time.localtime(t))
 292                 else:
 293                     return 'N/A'
 294
 295             tooltip = "last update: %(time)s\nbuild time: %(buildtime)s" % {
 296                 'time' : ftime(self.builders_status_time[b]),
 297                 'buildtime' : ftime(self.builders_status_buildtime[b]),
 298             }
 299             builders.append(link_pre +
 300                 "<font color='%(color)s'><b title=\"%(tooltip)s\">%(builder)s:%(status)s</b></font>" % {
 301                     'color' : c,
 302                     'builder' : b,
 303                     'status' : s,
 304                     'tooltip' : cgi.escape(tooltip, True),
 305             }
 306             + link_post)
 307         f.write("%s]</small></li>\n" % string.join(builders))
 308
 309     def rpmbuild_opts(self):
 310         """
 311             return all rpmbuild options related to this build
 312         """
 313         rpmopts = self.bconds_string() + self.kernel_string() + self.target_string() + self.defines_string()
 314         rpmdefs = \
 315             "--define '_topdir %(echo $HOME/rpm)' " \
 316             "--define '_specdir %{_topdir}/packages/%{name}' "  \
 317             "--define '_sourcedir %{_specdir}' " \
 318             "--define '_builddir %{_topdir}/BUILD/%{name}' "
 319         return rpmdefs + rpmopts
 320
 321     def kernel_string(self):
 322         r = ""
 323         if self.kernel != "":
 324             r = " --define 'alt_kernel " + self.kernel + "'"
 325         return r
 326
 327     def target_string(self):
 328         if len(self.target) > 0:
 329             return " --target " + ",".join(self.target)
 330         else:
 331             return ""
 332
 333     def bconds_string(self):
 334         r = ""
 335         for b in self.bconds_with:
 336             r = r + " --with " + b
 337         for b in self.bconds_without:
 338             r = r + " --without " + b
 339         return r
 340
 341     def defines_string(self):
 342         r = ""
 343         for key,value in self.defines.items():
 344             r += " --define '%s %s'" % (key, value)
 345         return r
 346
 347     def defines_xml(self):
 348         r = ""
 349         for key,value in self.defines.items():
 350             r += "<define name='%s'>%s</define>\n" % (escape(key), escape(value))
 351         return r
 352
 353     def default_target(self, arch):
 354         self.target.append("%s-pld-linux" % arch)
 355
 356     def write_to(self, f):
 357         f.write("""
 358          <batch id='%s' depends-on='%s'>
 359            <src-rpm>%s</src-rpm>
 360            <command flags="%s">%s</command>
 361            <spec>%s</spec>
 362            <branch>%s</branch>
 363            <info>%s</info>\n""" % (self.b_id,
 364                  string.join(map(lambda (b): b.b_id, self.depends_on)),
 365                  escape(self.src_rpm),
 366                  escape(' '.join(self.command_flags)), escape(self.command),
 367                  escape(self.spec), escape(self.branch), escape(self.info)))
 368         if self.kernel != "":
 369             f.write("           <kernel>%s</kernel>\n" % escape(self.kernel))
 370         for b in self.bconds_with:
 371             f.write("           <with>%s</with>\n" % escape(b))
 372         for b in self.target:
 373             f.write("           <target>%s</target>\n" % escape(b))
 374         for b in self.bconds_without:
 375             f.write("           <without>%s</without>\n" % escape(b))
 376         if self.defines:
 377             f.write("           %s\n" % self.defines_xml())
 378         for b in self.builders:
 379             if self.builders_status_buildtime.has_key(b):
 380                 t = self.builders_status_buildtime[b]
 381             else:
 382                 t = "0"
 383             f.write("           <builder status='%s' time='%s' buildtime='%s'>%s</builder>\n" % \
 384                     (escape(self.builders_status[b]), self.builders_status_time[b], t, escape(b)))
 385         f.write("         </batch>\n")
 386
 387     def log_line(self, l):
 388         log.notice(l)
 389         if self.logfile != None:
 390             util.append_to(self.logfile, l)
 391
 392     def expand_builders(batch, all_builders):
 393         all = []
 394         for bld in batch.builders:
 395             res = []
 396             for my_bld in all_builders:
 397                 if fnmatch.fnmatch(my_bld, bld):
 398                     res.append(my_bld)
 399             if res != []:
 400                 all.extend(res)
 401             else:
 402                 all.append(bld)
 403         batch.builders = all
 404
 405 class Notification:
 406     def __init__(self, e):
 407         self.batches = []
 408         self.kind = 'notification'
 409         self.group_id = attr(e, "group-id")
 410         self.builder = attr(e, "builder")
 411         self.batches = {}
 412         self.batches_buildtime = {}
 413         for c in e.childNodes:
 414             if is_blank(c): continue
 415             if c.nodeType != Element.ELEMENT_NODE:
 416                 log.panic("xml: evil notification child %d" % c.nodeType)
 417             if c.nodeName == "batch":
 418                 id = attr(c, "id")
 419                 status = attr(c, "status")
 420                 buildtime = attr(c, "buildtime", "0")
 421                 if not status.startswith("OK") and not status.startswith("SKIP") and not status.startswith("UNSUPP") and not status.startswith("FAIL"):
 422                     log.panic("xml notification: bad status: %s" % status)
 423                 self.batches[id] = status
 424                 self.batches_buildtime[id] = buildtime
 425             else:
 426                 log.panic("xml: evil notification child (%s)" % c.nodeName)
 427
 428     def apply_to(self, q):
 429         for r in q.requests:
 430             if r.kind == "group":
 431                 for b in r.batches:
 432                     if self.batches.has_key(b.b_id):
 433                         b.builders_status[self.builder] = self.batches[b.b_id]
 434                         b.builders_status_time[self.builder] = time.time()
 435                         b.builders_status_buildtime[self.builder] = "0" #self.batches_buildtime[b.b_id]
 436
 437 def build_request(e):
 438     if e.nodeType != Element.ELEMENT_NODE:
 439         log.panic("xml: evil request element")
 440     if e.nodeName == "group":
 441         return Group(e)
 442     elif e.nodeName == "notification":
 443         return Notification(e)
 444     elif e.nodeName == "command":
 445         # FIXME
 446         return Command(e)
 447     else:
 448         log.panic("xml: evil request [%s]" % e.nodeName)
 449
 450 def parse_request(f):
 451     d = parseString(f)
 452     return build_request(d.documentElement)
 453
 454 def parse_requests(f):
 455     d = parseString(f)
 456     res = []
 457     for r in d.documentElement.childNodes:
 458         if is_blank(r): continue
 459         res.append(build_request(r))
 460     return res