PLD_Builder/request.py

   1 # vi: encoding=utf-8 ts=8 sts=4 sw=4 et
   2
   3 from xml.dom.minidom import *
   4 from datetime import datetime
   5 import string
   6 import time
   7 import xml.sax.saxutils
   8 import fnmatch
   9 import os
  10 import urllib
  11 import cgi
  12 import pytz
  13 import tempfile
  14
  15 import util
  16 import log
  17 from acl import acl
  18 from config import config
  19
  20 __all__ = ['parse_request', 'parse_requests']
  21
  22 def text(e):
  23     res = ""
  24     for n in e.childNodes:
  25         if n.nodeType != Element.TEXT_NODE:
  26             log.panic("xml: text expected in <%s>, got %d" % (e.nodeName, n.nodeType))
  27         res += n.nodeValue
  28     return res
  29
  30 def attr(e, a, default = None):
  31     try:
  32         return e.attributes[a].value
  33     except:
  34         if default != None:
  35             return default
  36         raise
  37
  38 def escape(s):
  39     return xml.sax.saxutils.escape(s)
  40
  41 # return timestamp with timezone information
  42 # so we could parse it in javascript
  43 def tzdate(t):
  44     # as strftime %z is unofficial, and does not work, need to make it numeric ourselves
  45     date = time.strftime("%a %b %d %Y %H:%M:%S", time.localtime(t))
  46     # NOTE: the altzone is showing CURRENT timezone, not what the "t" reflects
  47     # NOTE: when DST is off timezone gets it right, altzone not
  48     if time.daylight:
  49         tzoffset = time.altzone
  50     else:
  51         tzoffset = time.timezone
  52     tz = '%+05d' % (-tzoffset / 3600 * 100)
  53     return date + ' ' + tz
  54
  55 # return date in iso8601 format
  56 def iso8601(ts, timezone='UTC'):
  57     tz = pytz.timezone(timezone)
  58     dt = datetime.fromtimestamp(ts, tz)
  59     return dt.isoformat()
  60
  61 def is_blank(e):
  62     return e.nodeType == Element.TEXT_NODE and string.strip(e.nodeValue) == ""
  63
  64 class Group:
  65     def __init__(self, e):
  66         self.batches = []
  67         self.kind = 'group'
  68         self.id = attr(e, "id")
  69         self.no = int(attr(e, "no"))
  70         self.priority = 2
  71         self.time = time.time()
  72         self.requester = ""
  73         self.max_jobs = 0
  74         self.requester_email = ""
  75         self.flags = string.split(attr(e, "flags", ""))
  76         for c in e.childNodes:
  77             if is_blank(c): continue
  78
  79             if c.nodeType != Element.ELEMENT_NODE:
  80                 log.panic("xml: evil group child %d" % c.nodeType)
  81             if c.nodeName == "batch":
  82                 self.batches.append(Batch(c))
  83             elif c.nodeName == "requester":
  84                 self.requester = text(c)
  85                 self.requester_email = attr(c, "email", "")
  86             elif c.nodeName == "priority":
  87                 self.priority = int(text(c))
  88             elif c.nodeName == "time":
  89                 self.time = int(text(c))
  90             elif c.nodeName == "maxjobs":
  91                 self.max_jobs = int(text(c))
  92             else:
  93                 log.panic("xml: evil group child (%s)" % c.nodeName)
  94         # note that we also check that group is sorted WRT deps
  95         m = {}
  96         for b in self.batches:
  97             deps = []
  98             m[b.b_id] = b
  99             for dep in b.depends_on:
 100                 if m.has_key(dep):
 101                     # avoid self-deps
 102                     if id(m[dep]) != id(b):
 103                         deps.append(m[dep])
 104                 else:
 105                     log.panic("xml: dependency not found in group")
 106             b.depends_on = deps
 107         if self.requester_email == "" and self.requester != "":
 108             self.requester_email = acl.user(self.requester).mail_to()
 109
 110     def dump(self, f):
 111         f.write("group: %d (id=%s pri=%d)\n" % (self.no, self.id, self.priority))
 112         f.write("  from: %s\n" % self.requester)
 113         f.write("  flags: %s\n" % string.join(self.flags))
 114         f.write("  time: %s\n" % time.asctime(time.localtime(self.time)))
 115         for b in self.batches:
 116             b.dump(f)
 117         f.write("\n")
 118
 119     # return structure usable for json encoding
 120     def dump_json(self):
 121         batches = []
 122         for b in self.batches:
 123             batches.append(b.dump_json())
 124
 125         return dict(
 126             no=self.no,
 127             id=self.id,
 128             time=self.time,
 129             requester=self.requester,
 130             priority=self.priority,
 131             max_jobs=self.max_jobs,
 132             flags=self.flags,
 133             batches=batches,
 134         )
 135
 136     def dump_html(self, f):
 137         f.write(
 138             "<div id=\"%(no)d\" class=\"request %(flags)s\">\n"
 139             "<a href=\"#%(no)d\">%(no)d</a>. "
 140             "<time class=\"timeago\" datetime=\"%(datetime)s\">%(time)s</time> "
 141             "from <b class=requester>%(requester)s</b> "
 142             "<small>%(id)s, prio=%(priority)d, jobs=%(max_jobs)d, %(flags)s</small>\n"
 143         % {
 144             'no': self.no,
 145             'id': '<a href="srpms/%(id)s">%(id)s</a>' % {'id': self.id},
 146             'time': escape(tzdate(self.time)),
 147             'datetime': escape(iso8601(self.time)),
 148             'requester': escape(self.requester),
 149             'priority': self.priority,
 150             'max_jobs': self.max_jobs,
 151             'flags': string.join(self.flags)
 152         })
 153         f.write("<ol>\n")
 154         for b in self.batches:
 155             b.dump_html(f, self.id)
 156         f.write("</ol>\n")
 157         f.write("</div>\n")
 158
 159     def write_to(self, f):
 160         f.write("""
 161        <group id="%s" no="%d" flags="%s">
 162          <requester email='%s'>%s</requester>
 163          <time>%d</time>
 164          <priority>%d</priority>
 165          <maxjobs>%d</maxjobs>\n""" % (self.id, self.no, string.join(self.flags),
 166                     escape(self.requester_email), escape(self.requester),
 167                     self.time, self.priority, self.max_jobs))
 168         for b in self.batches:
 169             b.write_to(f)
 170         f.write("       </group>\n\n")
 171
 172     def is_done(self):
 173         ok = 1
 174         for b in self.batches:
 175             if not b.is_done():
 176                 ok = 0
 177         return ok
 178
 179 class Batch:
 180     DEFAULT_PHP = '5.3'
 181
 182     def __init__(self, e):
 183         self.bconds_with = []
 184         self.bconds_without = []
 185         self.builders = []
 186         self.builders_status = {}
 187         self.builders_status_time = {}
 188         self.builders_status_buildtime = {}
 189         self.kernel = ""
 190         self.defines = {}
 191         self.target = []
 192         self.branch = ""
 193         self.src_rpm = ""
 194         self.info = ""
 195         self.spec = ""
 196         self.command = ""
 197         self.command_flags = []
 198         self.skip = []
 199         self.gb_id = ""
 200         self.b_id = attr(e, "id")
 201         self.depends_on = string.split(attr(e, "depends-on"))
 202         self.upgraded = True
 203
 204         self.parse_xml(e)
 205
 206         self.__topdir = None
 207
 208     def get_topdir(self):
 209         if not self.__topdir:
 210             self.__topdir = tempfile.mkdtemp(prefix='B.', dir='/tmp')
 211         return self.__topdir
 212
 213     def parse_xml(self, e):
 214         for c in e.childNodes:
 215             if is_blank(c): continue
 216
 217             if c.nodeType != Element.ELEMENT_NODE:
 218                 log.panic("xml: evil batch child %d" % c.nodeType)
 219             if c.nodeName == "src-rpm":
 220                 self.src_rpm = text(c)
 221             elif c.nodeName == "spec":
 222                 # normalize specname, specname is used as buildlog and we don't
 223                 # want to be exposed to directory traversal attacks
 224                 self.spec = text(c).split('/')[-1]
 225             elif c.nodeName == "command":
 226                 self.spec = "COMMAND"
 227                 self.command = text(c).strip()
 228                 self.command_flags = string.split(attr(c, "flags", ""))
 229             elif c.nodeName == "info":
 230                 self.info = text(c)
 231             elif c.nodeName == "kernel":
 232                 self.kernel = text(c)
 233             elif c.nodeName == "define":
 234                 define = attr(c, "name")
 235                 self.defines[define] = text(c)
 236             elif c.nodeName == "target":
 237                 self.target.append(text(c))
 238             elif c.nodeName == "skip":
 239                 self.skip.append(text(c))
 240             elif c.nodeName == "branch":
 241                 self.branch = text(c)
 242             elif c.nodeName == "builder":
 243                 key = text(c)
 244                 self.builders.append(key)
 245                 self.builders_status[key] = attr(c, "status", "?")
 246                 self.builders_status_time[key] = attr(c, "time", "0")
 247                 self.builders_status_buildtime[key] = "0" #attr(c, "buildtime", "0")
 248             elif c.nodeName == "with":
 249                 self.bconds_with.append(text(c))
 250             elif c.nodeName == "without":
 251                 self.bconds_without.append(text(c))
 252             else:
 253                 log.panic("xml: evil batch child (%s)" % c.nodeName)
 254
 255     def get_package_name(self):
 256         if len(self.spec) <= 5:
 257             return None
 258         return self.spec[:-5]
 259
 260     def tmpdir(self):
 261         """
 262         return tmpdir for this batch job building
 263         """
 264         # it's better to have TMPDIR and BUILD dir on same partition:
 265         # + /usr/bin/bzip2 -dc /home/services/builder/rpm/packages/kernel/patch-2.6.27.61.bz2
 266         # patch: **** Can't rename file /tmp/B.a1b1d3/poKWwRlp to drivers/scsi/hosts.c : No such file or directory
 267         path = os.path.join(self.get_topdir(), 'BUILD', 'tmp')
 268         return path
 269
 270     def is_done(self):
 271         ok = 1
 272         for b in self.builders:
 273             s = self.builders_status[b]
 274             if not s.startswith("OK") and not s.startswith("SKIP") and not s.startswith("UNSUPP") and not s.startswith("FAIL"):
 275                 ok = 0
 276         return ok
 277
 278     def dump(self, f):
 279         f.write("  batch: %s/%s\n" % (self.src_rpm, self.spec))
 280         f.write("    info: %s\n" % self.info)
 281         f.write("    kernel: %s\n" % self.kernel)
 282         f.write("    defines: %s\n" % self.defines_string())
 283         f.write("    target: %s\n" % self.target_string())
 284         f.write("    branch: %s\n" % self.branch)
 285         f.write("    bconds: %s\n" % self.bconds_string())
 286         builders = []
 287         for b in self.builders:
 288             builders.append("%s:%s" % (b, self.builders_status[b]))
 289         f.write("    builders: %s\n" % string.join(builders))
 290
 291     def is_command(self):
 292         return self.command != ""
 293
 294     # return structure usable for json encoding
 295     def dump_json(self):
 296         return dict(
 297             command=self.command,
 298             command_flags=self.command_flags,
 299
 300             spec=self.spec,
 301             branch=self.branch,
 302             package=self.spec[:-5],
 303             src_rpm=self.src_rpm,
 304
 305             bconds_with=self.bconds_with,
 306             bconds_without=self.bconds_without,
 307
 308             kernel=self.kernel,
 309             target=self.target,
 310             defines=self.defines,
 311
 312             builders=self.builders,
 313             builders_status=self.builders_status,
 314             builders_status_time=self.builders_status_time,
 315             builders_status_buildtime=self.builders_status_buildtime,
 316         )
 317
 318     def dump_html(self, f, rid):
 319         f.write("<li>\n")
 320         if self.is_command():
 321             desc = "SH: <pre>%s</pre> flags: [%s]" % (self.command, ' '.join(self.command_flags))
 322         else:
 323             package_url = "http://git.pld-linux.org/gitweb.cgi?p=packages/%(package)s.git;f=%(spec)s;h=%(branch)s;a=shortlog" % {
 324                 'spec': urllib.quote(self.spec),
 325                 'branch': urllib.quote(self.branch),
 326                 'package': urllib.quote(self.spec[:-5]),
 327             }
 328             desc = "%(src_rpm)s (<a href=\"%(package_url)s\">%(spec)s -r %(branch)s</a>%(rpmopts)s)" % {
 329                 'src_rpm': self.src_rpm,
 330                 'spec': self.spec,
 331                 'branch': self.branch,
 332                 'rpmopts': self.bconds_string() + self.kernel_string() + self.target_string() + self.defines_string(),
 333                 'package_url': package_url,
 334             }
 335         f.write("%s <small>[" % desc)
 336         builders = []
 337         for b in self.builders:
 338             s = self.builders_status[b]
 339             if s.startswith("OK"):
 340                 c = "green"
 341             elif s.startswith("FAIL"):
 342                 c = "red"
 343             elif s.startswith("SKIP"):
 344                 c = "blue"
 345             elif s.startswith("UNSUPP"):
 346                 c = "fuchsia"
 347             else:
 348                 c = "black"
 349             link_pre = ""
 350             link_post = ""
 351             if (s.startswith("OK") or s.startswith("SKIP") or s.startswith("UNSUPP") or s.startswith("FAIL")) and len(self.spec) > 5:
 352                 if self.is_command():
 353                     bl_name = "command"
 354                 else:
 355                     bl_name = self.spec[:len(self.spec)-5]
 356                 lin_ar = b.replace('noauto-','')
 357                 path = "/%s/%s/%s,%s.bz2" % (lin_ar.replace('-','/'), s, bl_name, rid)
 358                 is_ok = 0
 359                 if s.startswith("OK"):
 360                     is_ok = 1
 361                 bld = lin_ar.split('-')
 362                 tree_name = '-'.join(bld[:-1])
 363                 tree_arch = '-'.join(bld[-1:])
 364                 link_pre = "<a href=\"%s/index.php?dist=%s&arch=%s&ok=%d&name=%s&id=%s&action=tail\">" \
 365                         % (config.buildlogs, urllib.quote(tree_name), urllib.quote(tree_arch), is_ok, urllib.quote(bl_name), urllib.quote(rid))
 366                 link_post = "</a>"
 367
 368             def ftime(s):
 369                 t = float(s)
 370                 if t > 0:
 371                     return time.asctime(time.localtime(t))
 372                 else:
 373                     return 'N/A'
 374
 375             tooltip = "last update: %(time)s\nbuild time: %(buildtime)s" % {
 376                 'time' : ftime(self.builders_status_time[b]),
 377                 'buildtime' : ftime(self.builders_status_buildtime[b]),
 378             }
 379             builders.append(link_pre +
 380                 "<font color='%(color)s'><b title=\"%(tooltip)s\">%(builder)s:%(status)s</b></font>" % {
 381                     'color' : c,
 382                     'builder' : b,
 383                     'status' : s,
 384                     'tooltip' : cgi.escape(tooltip, True),
 385             }
 386             + link_post)
 387         f.write("%s]</small></li>\n" % string.join(builders))
 388
 389     def rpmbuild_opts(self):
 390         """
 391             return all rpmbuild options related to this build
 392         """
 393         rpmopts = self.bconds_string() + self.kernel_string() + self.target_string() + self.defines_string()
 394         rpmdefs = \
 395             "--define '_topdir %s' " % self.get_topdir() + \
 396             "--define '_specdir %{_topdir}' "  \
 397             "--define '_sourcedir %{_specdir}' " \
 398             "--define '_rpmdir %{_topdir}/RPMS' " \
 399             "--define '_builddir %{_topdir}/BUILD' "
 400         return rpmdefs + rpmopts
 401
 402     # transform php package name (52) to version (5.2)
 403     def php_name_to_ver(v):
 404         return '.'.join(list(v))
 405
 406     # transform php version (5.2) to package name (52)
 407     def php_ver_to_name(v):
 408         return v.replace('.', '')
 409
 410     def php_ignores(self, php_version):
 411         # available php versions in distro
 412         php_versions = ['4', '5.2', '5.3', '5.4', '5.5', '5.6', '7.0', '7.1', '7.2']
 413
 414         # remove current php version
 415         try:
 416             php_versions.remove(php_version)
 417         except ValueError:
 418             log.notice("Attempt to remove inexistent key '%s' from %s" % (php_version, php_versions))
 419             pass
 420
 421         # map them to poldek ignores
 422         # always ignore hhvm
 423         res = ['hhvm-*']
 424         for v in map(php_ver_to_name, php_versions):
 425             res.append("php%s-*" % v)
 426
 427         return res
 428
 429     # build ignore package list
 430     # currently only php ignore is filled based on build context
 431     def ignores(self):
 432         ignores = []
 433
 434         # add php version based ignores
 435         if self.defines.has_key('php_suffix'):
 436             # current version if -D php_suffix is present
 437             php_version = php_name_to_ver(self.defines['php_suffix'])
 438         else:
 439             php_version = self.DEFAULT_PHP
 440
 441         ignores.extend(self.php_ignores(php_version))
 442
 443         # return empty string if the list is empty
 444         if len(ignores) == 0:
 445             return ""
 446
 447         def add_ignore(s):
 448             return "--ignore=%s" % s
 449
 450         return " ".join(map(add_ignore, ignores))
 451
 452     def kernel_string(self):
 453         r = ""
 454         if self.kernel != "":
 455             r = " --define 'alt_kernel " + self.kernel + "'"
 456         return r
 457
 458     def target_string(self):
 459         if len(self.target) > 0:
 460             return " --target " + ",".join(self.target)
 461         else:
 462             return ""
 463
 464     def bconds_string(self):
 465         r = ""
 466         for b in self.bconds_with:
 467             r = r + " --with " + b
 468         for b in self.bconds_without:
 469             r = r + " --without " + b
 470         return r
 471
 472     def defines_string(self):
 473         r = ""
 474         for key,value in self.defines.items():
 475             r += " --define '%s %s'" % (key, value)
 476         return r
 477
 478     def defines_xml(self):
 479         r = ""
 480         for key,value in self.defines.items():
 481             r += "<define name='%s'>%s</define>\n" % (escape(key), escape(value))
 482         return r
 483
 484     def default_target(self, arch):
 485         self.target.append("%s-pld-linux" % arch)
 486
 487     def write_to(self, f):
 488         f.write("""
 489          <batch id='%s' depends-on='%s'>
 490            <src-rpm>%s</src-rpm>
 491            <command flags="%s">%s</command>
 492            <spec>%s</spec>
 493            <branch>%s</branch>
 494            <info>%s</info>\n""" % (self.b_id,
 495                  string.join(map(lambda (b): b.b_id, self.depends_on)),
 496                  escape(self.src_rpm),
 497                  escape(' '.join(self.command_flags)), escape(self.command),
 498                  escape(self.spec), escape(self.branch), escape(self.info)))
 499         if self.kernel != "":
 500             f.write("           <kernel>%s</kernel>\n" % escape(self.kernel))
 501         for b in self.bconds_with:
 502             f.write("           <with>%s</with>\n" % escape(b))
 503         for b in self.target:
 504             f.write("           <target>%s</target>\n" % escape(b))
 505         for b in self.bconds_without:
 506             f.write("           <without>%s</without>\n" % escape(b))
 507         if self.defines:
 508             f.write("           %s\n" % self.defines_xml())
 509         for b in self.builders:
 510             if self.builders_status_buildtime.has_key(b):
 511                 t = self.builders_status_buildtime[b]
 512             else:
 513                 t = "0"
 514             f.write("           <builder status='%s' time='%s' buildtime='%s'>%s</builder>\n" % \
 515                     (escape(self.builders_status[b]), self.builders_status_time[b], t, escape(b)))
 516         f.write("         </batch>\n")
 517
 518     def log_line(self, l):
 519         log.notice(l)
 520         if self.logfile != None:
 521             util.append_to(self.logfile, l)
 522
 523     def expand_builders(batch, all_builders):
 524         all = []
 525         for bld in batch.builders:
 526             res = []
 527             for my_bld in all_builders:
 528                 if fnmatch.fnmatch(my_bld, bld):
 529                     res.append(my_bld)
 530             if res != []:
 531                 all.extend(res)
 532             else:
 533                 all.append(bld)
 534         batch.builders = all
 535
 536 class Notification:
 537     def __init__(self, e):
 538         self.batches = []
 539         self.kind = 'notification'
 540         self.group_id = attr(e, "group-id")
 541         self.builder = attr(e, "builder")
 542         self.batches = {}
 543         self.batches_buildtime = {}
 544         for c in e.childNodes:
 545             if is_blank(c): continue
 546             if c.nodeType != Element.ELEMENT_NODE:
 547                 log.panic("xml: evil notification child %d" % c.nodeType)
 548             if c.nodeName == "batch":
 549                 id = attr(c, "id")
 550                 status = attr(c, "status")
 551                 buildtime = attr(c, "buildtime", "0")
 552                 if not status.startswith("OK") and not status.startswith("SKIP") and not status.startswith("UNSUPP") and not status.startswith("FAIL"):
 553                     log.panic("xml notification: bad status: %s" % status)
 554                 self.batches[id] = status
 555                 self.batches_buildtime[id] = buildtime
 556             else:
 557                 log.panic("xml: evil notification child (%s)" % c.nodeName)
 558
 559     # return structure usable for json encoding
 560     def dump_json(self):
 561         return dict(
 562             id=self.group_id,
 563             builder=self.builder,
 564             batches=self.batches,
 565             batches_buildtime=self.batches_buildtime,
 566         )
 567
 568     def apply_to(self, q):
 569         for r in q.requests:
 570             if r.kind == "group":
 571                 for b in r.batches:
 572                     if self.batches.has_key(b.b_id):
 573                         b.builders_status[self.builder] = self.batches[b.b_id]
 574                         b.builders_status_time[self.builder] = time.time()
 575                         b.builders_status_buildtime[self.builder] = "0" #self.batches_buildtime[b.b_id]
 576
 577 def build_request(e):
 578     if e.nodeType != Element.ELEMENT_NODE:
 579         log.panic("xml: evil request element")
 580     if e.nodeName == "group":
 581         return Group(e)
 582     elif e.nodeName == "notification":
 583         return Notification(e)
 584     elif e.nodeName == "command":
 585         # FIXME
 586         return Command(e)
 587     else:
 588         log.panic("xml: evil request [%s]" % e.nodeName)
 589
 590 def parse_request(f):
 591     d = parseString(f)
 592     return build_request(d.documentElement)
 593
 594 def parse_requests(f):
 595     d = parseString(f)
 596     res = []
 597     for r in d.documentElement.childNodes:
 598         if is_blank(r): continue
 599         res.append(build_request(r))
 600     return res