PLD_Builder/request.py

   1 # vi: encoding=utf-8 ts=8 sts=4 sw=4 et
   2
   3 from xml.dom.minidom import *
   4 from datetime import datetime
   5 import string
   6 import time
   7 import xml.sax.saxutils
   8 import fnmatch
   9 import os
  10 import urllib
  11 import cgi
  12 import pytz
  13
  14 import util
  15 import log
  16 from acl import acl
  17 from config import config
  18
  19 __all__ = ['parse_request', 'parse_requests']
  20
  21 def text(e):
  22     res = ""
  23     for n in e.childNodes:
  24         if n.nodeType != Element.TEXT_NODE:
  25             log.panic("xml: text expected in <%s>, got %d" % (e.nodeName, n.nodeType))
  26         res += n.nodeValue
  27     return res
  28
  29 def attr(e, a, default = None):
  30     try:
  31         return e.attributes[a].value
  32     except:
  33         if default != None:
  34             return default
  35         raise
  36
  37 def escape(s):
  38     return xml.sax.saxutils.escape(s)
  39
  40 # return timestamp with timezone information
  41 # so we could parse it in javascript
  42 def tzdate(t):
  43     # as strftime %z is unofficial, and does not work, need to make it numeric ourselves
  44     date = time.strftime("%a %b %d %Y %H:%M:%S", time.localtime(t))
  45     # NOTE: the altzone is showing CURRENT timezone, not what the "t" reflects
  46     # NOTE: when DST is off timezone gets it right, altzone not
  47     if time.daylight:
  48         tzoffset = time.altzone
  49     else:
  50         tzoffset = time.timezone
  51     tz = '%+05d' % (-tzoffset / 3600 * 100)
  52     return date + ' ' + tz
  53
  54 # return date in iso8601 format
  55 def iso8601(ts, timezone='UTC'):
  56     tz = pytz.timezone(timezone)
  57     dt = datetime.fromtimestamp(ts, tz)
  58     return dt.isoformat()
  59
  60 def is_blank(e):
  61     return e.nodeType == Element.TEXT_NODE and string.strip(e.nodeValue) == ""
  62
  63 class Group:
  64     def __init__(self, e):
  65         self.batches = []
  66         self.kind = 'group'
  67         self.id = attr(e, "id")
  68         self.no = int(attr(e, "no"))
  69         self.priority = 2
  70         self.time = time.time()
  71         self.requester = ""
  72         self.max_jobs = 0
  73         self.requester_email = ""
  74         self.flags = string.split(attr(e, "flags", ""))
  75         for c in e.childNodes:
  76             if is_blank(c): continue
  77
  78             if c.nodeType != Element.ELEMENT_NODE:
  79                 log.panic("xml: evil group child %d" % c.nodeType)
  80             if c.nodeName == "batch":
  81                 self.batches.append(Batch(c))
  82             elif c.nodeName == "requester":
  83                 self.requester = text(c)
  84                 self.requester_email = attr(c, "email", "")
  85             elif c.nodeName == "priority":
  86                 self.priority = int(text(c))
  87             elif c.nodeName == "time":
  88                 self.time = int(text(c))
  89             elif c.nodeName == "maxjobs":
  90                 self.max_jobs = int(text(c))
  91             else:
  92                 log.panic("xml: evil group child (%s)" % c.nodeName)
  93         # note that we also check that group is sorted WRT deps
  94         m = {}
  95         for b in self.batches:
  96             deps = []
  97             m[b.b_id] = b
  98             for dep in b.depends_on:
  99                 if m.has_key(dep):
 100                     # avoid self-deps
 101                     if id(m[dep]) != id(b):
 102                         deps.append(m[dep])
 103                 else:
 104                     log.panic("xml: dependency not found in group")
 105             b.depends_on = deps
 106         if self.requester_email == "" and self.requester != "":
 107             self.requester_email = acl.user(self.requester).mail_to()
 108
 109     def dump(self, f):
 110         f.write("group: %d (id=%s pri=%d)\n" % (self.no, self.id, self.priority))
 111         f.write("  from: %s\n" % self.requester)
 112         f.write("  flags: %s\n" % string.join(self.flags))
 113         f.write("  time: %s\n" % time.asctime(time.localtime(self.time)))
 114         for b in self.batches:
 115             b.dump(f)
 116         f.write("\n")
 117
 118     def dump_html(self, f):
 119         f.write(
 120             "<div id=\"%(no)d\" class=\"request %(flags)s\">\n"
 121             "<a href=\"#%(no)d\">%(no)d</a>. "
 122             "<time class=\"timeago\" datetime=\"%(datetime)s\">%(time)s</time> "
 123             "from <b class=requester>%(requester)s</b> "
 124             "<small>%(id)s, prio=%(priority)d, jobs=%(max_jobs)d, %(flags)s</small>\n"
 125         % {
 126             'no': self.no,
 127             'id': '<a href="srpms/%(id)s">%(id)s</a>' % {'id': self.id},
 128             'time': escape(tzdate(self.time)),
 129             'datetime': escape(iso8601(self.time)),
 130             'requester': escape(self.requester),
 131             'priority': self.priority,
 132             'max_jobs': self.max_jobs,
 133             'flags': string.join(self.flags)
 134         })
 135         f.write("<ol>\n")
 136         for b in self.batches:
 137             b.dump_html(f, self.id)
 138         f.write("</ol>\n")
 139         f.write("</div>\n")
 140
 141     def write_to(self, f):
 142         f.write("""
 143        <group id="%s" no="%d" flags="%s">
 144          <requester email='%s'>%s</requester>
 145          <time>%d</time>
 146          <priority>%d</priority>
 147          <maxjobs>%d</maxjobs>\n""" % (self.id, self.no, string.join(self.flags),
 148                     escape(self.requester_email), escape(self.requester),
 149                     self.time, self.priority, self.max_jobs))
 150         for b in self.batches:
 151             b.write_to(f)
 152         f.write("       </group>\n\n")
 153
 154     def is_done(self):
 155         ok = 1
 156         for b in self.batches:
 157             if not b.is_done():
 158                 ok = 0
 159         return ok
 160
 161 class Batch:
 162     def __init__(self, e):
 163         self.bconds_with = []
 164         self.bconds_without = []
 165         self.builders = []
 166         self.builders_status = {}
 167         self.builders_status_time = {}
 168         self.builders_status_buildtime = {}
 169         self.kernel = ""
 170         self.defines = {}
 171         self.target = []
 172         self.branch = ""
 173         self.src_rpm = ""
 174         self.info = ""
 175         self.spec = ""
 176         self.command = ""
 177         self.command_flags = []
 178         self.skip = []
 179         self.gb_id = ""
 180         self.b_id = attr(e, "id")
 181         self.depends_on = string.split(attr(e, "depends-on"))
 182         self.upgraded = True
 183
 184         self.parse_xml(e)
 185
 186         self._topdir = '/tmp/B.%s' % self.b_id
 187
 188     def parse_xml(self, e):
 189         for c in e.childNodes:
 190             if is_blank(c): continue
 191
 192             if c.nodeType != Element.ELEMENT_NODE:
 193                 log.panic("xml: evil batch child %d" % c.nodeType)
 194             if c.nodeName == "src-rpm":
 195                 self.src_rpm = text(c)
 196             elif c.nodeName == "spec":
 197                 # normalize specname, specname is used as buildlog and we don't
 198                 # want to be exposed to directory traversal attacks
 199                 self.spec = text(c).split('/')[-1]
 200             elif c.nodeName == "command":
 201                 self.spec = "COMMAND"
 202                 self.command = text(c).strip()
 203                 self.command_flags = string.split(attr(c, "flags", ""))
 204             elif c.nodeName == "info":
 205                 self.info = text(c)
 206             elif c.nodeName == "kernel":
 207                 self.kernel = text(c)
 208             elif c.nodeName == "define":
 209                 define = attr(c, "name")
 210                 self.defines[define] = text(c)
 211             elif c.nodeName == "target":
 212                 self.target.append(text(c))
 213             elif c.nodeName == "skip":
 214                 self.skip.append(text(c))
 215             elif c.nodeName == "branch":
 216                 self.branch = text(c)
 217             elif c.nodeName == "builder":
 218                 key = text(c)
 219                 self.builders.append(key)
 220                 self.builders_status[key] = attr(c, "status", "?")
 221                 self.builders_status_time[key] = attr(c, "time", "0")
 222                 self.builders_status_buildtime[key] = "0" #attr(c, "buildtime", "0")
 223             elif c.nodeName == "with":
 224                 self.bconds_with.append(text(c))
 225             elif c.nodeName == "without":
 226                 self.bconds_without.append(text(c))
 227             else:
 228                 log.panic("xml: evil batch child (%s)" % c.nodeName)
 229
 230     def get_package_name(self):
 231         if len(self.spec) <= 5:
 232             return None
 233         return self.spec[:-5]
 234
 235     def tmpdir(self):
 236         """
 237         return tmpdir for this batch job building
 238         """
 239         # it's better to have TMPDIR and BUILD dir on same partition:
 240         # + /usr/bin/bzip2 -dc /home/services/builder/rpm/packages/kernel/patch-2.6.27.61.bz2
 241         # patch: **** Can't rename file /tmp/B.a1b1d3/poKWwRlp to drivers/scsi/hosts.c : No such file or directory
 242         path = os.path.join(self._topdir, 'BUILD', 'tmp')
 243         return path
 244
 245     def is_done(self):
 246         ok = 1
 247         for b in self.builders:
 248             s = self.builders_status[b]
 249             if not s.startswith("OK") and not s.startswith("SKIP") and not s.startswith("UNSUPP") and not s.startswith("FAIL"):
 250                 ok = 0
 251         return ok
 252
 253     def dump(self, f):
 254         f.write("  batch: %s/%s\n" % (self.src_rpm, self.spec))
 255         f.write("    info: %s\n" % self.info)
 256         f.write("    kernel: %s\n" % self.kernel)
 257         f.write("    defines: %s\n" % self.defines_string())
 258         f.write("    target: %s\n" % self.target_string())
 259         f.write("    branch: %s\n" % self.branch)
 260         f.write("    bconds: %s\n" % self.bconds_string())
 261         builders = []
 262         for b in self.builders:
 263             builders.append("%s:%s" % (b, self.builders_status[b]))
 264         f.write("    builders: %s\n" % string.join(builders))
 265
 266     def is_command(self):
 267         return self.command != ""
 268
 269     def dump_html(self, f, rid):
 270         f.write("<li>\n")
 271         if self.is_command():
 272             desc = "SH: <pre>%s</pre> flags: [%s]" % (self.command, ' '.join(self.command_flags))
 273         else:
 274             package_url = "http://git.pld-linux.org/gitweb.cgi?p=packages/%(package)s.git;f=%(spec)s;h=%(branch)s;a=shortlog" % {
 275                 'spec': urllib.quote(self.spec),
 276                 'branch': urllib.quote(self.branch),
 277                 'package': urllib.quote(self.spec[:-5]),
 278             }
 279             desc = "%(src_rpm)s (<a href=\"%(package_url)s\">%(spec)s -r %(branch)s</a>%(rpmopts)s)" % {
 280                 'src_rpm': self.src_rpm,
 281                 'spec': self.spec,
 282                 'branch': self.branch,
 283                 'rpmopts': self.bconds_string() + self.kernel_string() + self.target_string() + self.defines_string(),
 284                 'package_url': package_url,
 285             }
 286         f.write("%s <small>[" % desc)
 287         builders = []
 288         for b in self.builders:
 289             s = self.builders_status[b]
 290             if s.startswith("OK"):
 291                 c = "green"
 292             elif s.startswith("FAIL"):
 293                 c = "red"
 294             elif s.startswith("SKIP"):
 295                 c = "blue"
 296             elif s.startswith("UNSUPP"):
 297                 c = "fuchsia"
 298             else:
 299                 c = "black"
 300             link_pre = ""
 301             link_post = ""
 302             if (s.startswith("OK") or s.startswith("SKIP") or s.startswith("UNSUPP") or s.startswith("FAIL")) and len(self.spec) > 5:
 303                 if self.is_command():
 304                     bl_name = "command"
 305                 else:
 306                     bl_name = self.spec[:len(self.spec)-5]
 307                 lin_ar = b.replace('noauto-','')
 308                 path = "/%s/%s/%s,%s.bz2" % (lin_ar.replace('-','/'), s, bl_name, rid)
 309                 is_ok = 0
 310                 if s.startswith("OK"):
 311                     is_ok = 1
 312                 bld = lin_ar.split('-')
 313                 tree_name = '-'.join(bld[:-1])
 314                 tree_arch = '-'.join(bld[-1:])
 315                 link_pre = "<a href=\"%s/index.php?dist=%s&arch=%s&ok=%d&name=%s&id=%s&action=tail\">" \
 316                         % (config.buildlogs, urllib.quote(tree_name), urllib.quote(tree_arch), is_ok, urllib.quote(bl_name), urllib.quote(rid))
 317                 link_post = "</a>"
 318
 319             def ftime(s):
 320                 t = float(s)
 321                 if t > 0:
 322                     return time.asctime(time.localtime(t))
 323                 else:
 324                     return 'N/A'
 325
 326             tooltip = "last update: %(time)s\nbuild time: %(buildtime)s" % {
 327                 'time' : ftime(self.builders_status_time[b]),
 328                 'buildtime' : ftime(self.builders_status_buildtime[b]),
 329             }
 330             builders.append(link_pre +
 331                 "<font color='%(color)s'><b title=\"%(tooltip)s\">%(builder)s:%(status)s</b></font>" % {
 332                     'color' : c,
 333                     'builder' : b,
 334                     'status' : s,
 335                     'tooltip' : cgi.escape(tooltip, True),
 336             }
 337             + link_post)
 338         f.write("%s]</small></li>\n" % string.join(builders))
 339
 340     def rpmbuild_opts(self):
 341         """
 342             return all rpmbuild options related to this build
 343         """
 344         rpmopts = self.bconds_string() + self.kernel_string() + self.target_string() + self.defines_string()
 345         rpmdefs = \
 346             "--define '_topdir %s' " % self._topdir + \
 347             "--define '_specdir %{_topdir}' "  \
 348             "--define '_sourcedir %{_specdir}' " \
 349             "--define '_rpmdir %{_topdir}/RPMS' " \
 350             "--define '_builddir %{_topdir}/BUILD' "
 351         return rpmdefs + rpmopts
 352
 353     def php_ignores(self):
 354         # transform php package name (52) to version (5.2)
 355         def php_name_to_ver(v):
 356             return '.'.join(list(v))
 357
 358         # transform php version (5.2) to package name (52)
 359         def php_ver_to_name(v):
 360             return v.replace('.', '')
 361
 362         # available php versions in distro
 363         php_versions = ['4', '5.2', '5.3', '5.4', '5.5', '5.6', '7.0']
 364
 365         # current version if -D php_suffix is present
 366         php_version = php_name_to_ver(self.defines['php_suffix'])
 367
 368         # remove current php version
 369         try:
 370             php_versions.remove(php_version)
 371         except ValueError:
 372             log.notice("Attempt to remove inexistent key '%s' from %s" % (php_version, php_versions))
 373             pass
 374
 375         # map them to poldek ignores
 376         # always ignore hhvm
 377         res = ['hhvm']
 378         for v in map(php_ver_to_name, php_versions):
 379             res.append("php%s-*" % v)
 380
 381         return res
 382
 383     # build ignore package list
 384     # currently only php ignore is filled based on build context
 385     def ignores(self):
 386         ignores = []
 387
 388         # add php version based ignores
 389         if self.defines.has_key('php_suffix'):
 390             ignores.extend(self.php_ignores())
 391
 392         # return empty string if the list is empty
 393         if len(ignores) == 0:
 394             return ""
 395
 396         def add_ignore(s):
 397             return "--ignore=%s" % s
 398
 399         return " ".join(map(add_ignore, ignores))
 400
 401     def kernel_string(self):
 402         r = ""
 403         if self.kernel != "":
 404             r = " --define 'alt_kernel " + self.kernel + "'"
 405         return r
 406
 407     def target_string(self):
 408         if len(self.target) > 0:
 409             return " --target " + ",".join(self.target)
 410         else:
 411             return ""
 412
 413     def bconds_string(self):
 414         r = ""
 415         for b in self.bconds_with:
 416             r = r + " --with " + b
 417         for b in self.bconds_without:
 418             r = r + " --without " + b
 419         return r
 420
 421     def defines_string(self):
 422         r = ""
 423         for key,value in self.defines.items():
 424             r += " --define '%s %s'" % (key, value)
 425         return r
 426
 427     def defines_xml(self):
 428         r = ""
 429         for key,value in self.defines.items():
 430             r += "<define name='%s'>%s</define>\n" % (escape(key), escape(value))
 431         return r
 432
 433     def default_target(self, arch):
 434         self.target.append("%s-pld-linux" % arch)
 435
 436     def write_to(self, f):
 437         f.write("""
 438          <batch id='%s' depends-on='%s'>
 439            <src-rpm>%s</src-rpm>
 440            <command flags="%s">%s</command>
 441            <spec>%s</spec>
 442            <branch>%s</branch>
 443            <info>%s</info>\n""" % (self.b_id,
 444                  string.join(map(lambda (b): b.b_id, self.depends_on)),
 445                  escape(self.src_rpm),
 446                  escape(' '.join(self.command_flags)), escape(self.command),
 447                  escape(self.spec), escape(self.branch), escape(self.info)))
 448         if self.kernel != "":
 449             f.write("           <kernel>%s</kernel>\n" % escape(self.kernel))
 450         for b in self.bconds_with:
 451             f.write("           <with>%s</with>\n" % escape(b))
 452         for b in self.target:
 453             f.write("           <target>%s</target>\n" % escape(b))
 454         for b in self.bconds_without:
 455             f.write("           <without>%s</without>\n" % escape(b))
 456         if self.defines:
 457             f.write("           %s\n" % self.defines_xml())
 458         for b in self.builders:
 459             if self.builders_status_buildtime.has_key(b):
 460                 t = self.builders_status_buildtime[b]
 461             else:
 462                 t = "0"
 463             f.write("           <builder status='%s' time='%s' buildtime='%s'>%s</builder>\n" % \
 464                     (escape(self.builders_status[b]), self.builders_status_time[b], t, escape(b)))
 465         f.write("         </batch>\n")
 466
 467     def log_line(self, l):
 468         log.notice(l)
 469         if self.logfile != None:
 470             util.append_to(self.logfile, l)
 471
 472     def expand_builders(batch, all_builders):
 473         all = []
 474         for bld in batch.builders:
 475             res = []
 476             for my_bld in all_builders:
 477                 if fnmatch.fnmatch(my_bld, bld):
 478                     res.append(my_bld)
 479             if res != []:
 480                 all.extend(res)
 481             else:
 482                 all.append(bld)
 483         batch.builders = all
 484
 485 class Notification:
 486     def __init__(self, e):
 487         self.batches = []
 488         self.kind = 'notification'
 489         self.group_id = attr(e, "group-id")
 490         self.builder = attr(e, "builder")
 491         self.batches = {}
 492         self.batches_buildtime = {}
 493         for c in e.childNodes:
 494             if is_blank(c): continue
 495             if c.nodeType != Element.ELEMENT_NODE:
 496                 log.panic("xml: evil notification child %d" % c.nodeType)
 497             if c.nodeName == "batch":
 498                 id = attr(c, "id")
 499                 status = attr(c, "status")
 500                 buildtime = attr(c, "buildtime", "0")
 501                 if not status.startswith("OK") and not status.startswith("SKIP") and not status.startswith("UNSUPP") and not status.startswith("FAIL"):
 502                     log.panic("xml notification: bad status: %s" % status)
 503                 self.batches[id] = status
 504                 self.batches_buildtime[id] = buildtime
 505             else:
 506                 log.panic("xml: evil notification child (%s)" % c.nodeName)
 507
 508     def apply_to(self, q):
 509         for r in q.requests:
 510             if r.kind == "group":
 511                 for b in r.batches:
 512                     if self.batches.has_key(b.b_id):
 513                         b.builders_status[self.builder] = self.batches[b.b_id]
 514                         b.builders_status_time[self.builder] = time.time()
 515                         b.builders_status_buildtime[self.builder] = "0" #self.batches_buildtime[b.b_id]
 516
 517 def build_request(e):
 518     if e.nodeType != Element.ELEMENT_NODE:
 519         log.panic("xml: evil request element")
 520     if e.nodeName == "group":
 521         return Group(e)
 522     elif e.nodeName == "notification":
 523         return Notification(e)
 524     elif e.nodeName == "command":
 525         # FIXME
 526         return Command(e)
 527     else:
 528         log.panic("xml: evil request [%s]" % e.nodeName)
 529
 530 def parse_request(f):
 531     d = parseString(f)
 532     return build_request(d.documentElement)
 533
 534 def parse_requests(f):
 535     d = parseString(f)
 536     res = []
 537     for r in d.documentElement.childNodes:
 538         if is_blank(r): continue
 539         res.append(build_request(r))
 540     return res