PLD_Builder/request.py

   1 # vi: encoding=utf-8 ts=8 sts=4 sw=4 et
   2
   3 from xml.dom.minidom import *
   4 from datetime import datetime
   5 import string
   6 import time
   7 import xml.sax.saxutils
   8 import fnmatch
   9 import os
  10 import urllib
  11 import cgi
  12 import pytz
  13 import tempfile
  14
  15 import util
  16 import log
  17 from acl import acl
  18 from config import config
  19
  20 __all__ = ['parse_request', 'parse_requests']
  21
  22 def text(e):
  23     res = ""
  24     for n in e.childNodes:
  25         if n.nodeType != Element.TEXT_NODE:
  26             log.panic("xml: text expected in <%s>, got %d" % (e.nodeName, n.nodeType))
  27         res += n.nodeValue
  28     return res
  29
  30 def attr(e, a, default = None):
  31     try:
  32         return e.attributes[a].value
  33     except:
  34         if default != None:
  35             return default
  36         raise
  37
  38 def escape(s):
  39     return xml.sax.saxutils.escape(s)
  40
  41 # return timestamp with timezone information
  42 # so we could parse it in javascript
  43 def tzdate(t):
  44     # as strftime %z is unofficial, and does not work, need to make it numeric ourselves
  45     date = time.strftime("%a %b %d %Y %H:%M:%S", time.localtime(t))
  46     # NOTE: the altzone is showing CURRENT timezone, not what the "t" reflects
  47     # NOTE: when DST is off timezone gets it right, altzone not
  48     if time.daylight:
  49         tzoffset = time.altzone
  50     else:
  51         tzoffset = time.timezone
  52     tz = '%+05d' % (-tzoffset / 3600 * 100)
  53     return date + ' ' + tz
  54
  55 # return date in iso8601 format
  56 def iso8601(ts, timezone='UTC'):
  57     tz = pytz.timezone(timezone)
  58     dt = datetime.fromtimestamp(ts, tz)
  59     return dt.isoformat()
  60
  61 def is_blank(e):
  62     return e.nodeType == Element.TEXT_NODE and string.strip(e.nodeValue) == ""
  63
  64 class Group:
  65     def __init__(self, e):
  66         self.batches = []
  67         self.kind = 'group'
  68         self.id = attr(e, "id")
  69         self.no = int(attr(e, "no"))
  70         self.priority = 2
  71         self.time = time.time()
  72         self.requester = ""
  73         self.max_jobs = 0
  74         self.requester_email = ""
  75         self.flags = string.split(attr(e, "flags", ""))
  76         for c in e.childNodes:
  77             if is_blank(c): continue
  78
  79             if c.nodeType != Element.ELEMENT_NODE:
  80                 log.panic("xml: evil group child %d" % c.nodeType)
  81             if c.nodeName == "batch":
  82                 self.batches.append(Batch(c))
  83             elif c.nodeName == "requester":
  84                 self.requester = text(c)
  85                 self.requester_email = attr(c, "email", "")
  86             elif c.nodeName == "priority":
  87                 self.priority = int(text(c))
  88             elif c.nodeName == "time":
  89                 self.time = int(text(c))
  90             elif c.nodeName == "maxjobs":
  91                 self.max_jobs = int(text(c))
  92             else:
  93                 log.panic("xml: evil group child (%s)" % c.nodeName)
  94         # note that we also check that group is sorted WRT deps
  95         m = {}
  96         for b in self.batches:
  97             deps = []
  98             m[b.b_id] = b
  99             for dep in b.depends_on:
 100                 if m.has_key(dep):
 101                     # avoid self-deps
 102                     if id(m[dep]) != id(b):
 103                         deps.append(m[dep])
 104                 else:
 105                     log.panic("xml: dependency not found in group")
 106             b.depends_on = deps
 107         if self.requester_email == "" and self.requester != "":
 108             self.requester_email = acl.user(self.requester).mail_to()
 109
 110     def dump(self, f):
 111         f.write("group: %d (id=%s pri=%d)\n" % (self.no, self.id, self.priority))
 112         f.write("  from: %s\n" % self.requester)
 113         f.write("  flags: %s\n" % string.join(self.flags))
 114         f.write("  time: %s\n" % time.asctime(time.localtime(self.time)))
 115         for b in self.batches:
 116             b.dump(f)
 117         f.write("\n")
 118
 119     # return structure usable for json encoding
 120     def dump_json(self):
 121         batches = []
 122         for b in self.batches:
 123             batches.append(b.dump_json())
 124
 125         return dict(
 126             no=self.no,
 127             id=self.id,
 128             time=self.time,
 129             requester=self.requester,
 130             priority=self.priority,
 131             max_jobs=self.max_jobs,
 132             flags=self.flags,
 133             batches=batches,
 134         )
 135
 136     def dump_html(self, f):
 137         f.write(
 138             "<div id=\"%(no)d\" class=\"request %(flags)s\">\n"
 139             "<a href=\"#%(no)d\">%(no)d</a>. "
 140             "<time class=\"timeago\" title=\"%(datetime)s\" datetime=\"%(datetime)s\">%(time)s</time> "
 141             "from <b class=requester>%(requester)s</b> "
 142             "<small>%(id)s, prio=%(priority)d, jobs=%(max_jobs)d, %(flags)s</small>\n"
 143         % {
 144             'no': self.no,
 145             'id': '<a href="srpms/%(id)s">%(id)s</a>' % {'id': self.id},
 146             'time': escape(tzdate(self.time)),
 147             'datetime': escape(iso8601(self.time)),
 148             'requester': escape(self.requester),
 149             'priority': self.priority,
 150             'max_jobs': self.max_jobs,
 151             'flags': string.join(self.flags)
 152         })
 153         f.write("<ol>\n")
 154         for b in self.batches:
 155             b.dump_html(f, self.id)
 156         f.write("</ol>\n")
 157         f.write("</div>\n")
 158
 159     def write_to(self, f):
 160         f.write("""
 161        <group id="%s" no="%d" flags="%s">
 162          <requester email='%s'>%s</requester>
 163          <time>%d</time>
 164          <priority>%d</priority>
 165          <maxjobs>%d</maxjobs>\n""" % (self.id, self.no, string.join(self.flags),
 166                     escape(self.requester_email), escape(self.requester),
 167                     self.time, self.priority, self.max_jobs))
 168         for b in self.batches:
 169             b.write_to(f)
 170         f.write("       </group>\n\n")
 171
 172     def is_done(self):
 173         ok = 1
 174         for b in self.batches:
 175             if not b.is_done():
 176                 ok = 0
 177         return ok
 178
 179 # transform php package name (52) to version (5.2)
 180 def php_name_to_ver(v):
 181     return '.'.join(list(v))
 182
 183 # transform php version (5.2) to package name (52)
 184 def php_ver_to_name(v):
 185     return v.replace('.', '')
 186
 187 class Batch:
 188     DEFAULT_PHP = '5.3'
 189
 190     def __init__(self, e):
 191         self.bconds_with = []
 192         self.bconds_without = []
 193         self.builders = []
 194         self.builders_status = {}
 195         self.builders_status_time = {}
 196         self.builders_status_buildtime = {}
 197         self.kernel = ""
 198         self.defines = {}
 199         self.target = []
 200         self.branch = ""
 201         self.src_rpm = ""
 202         self.info = ""
 203         self.spec = ""
 204         self.command = ""
 205         self.command_flags = []
 206         self.skip = []
 207         self.gb_id = ""
 208         self.b_id = attr(e, "id")
 209         self.depends_on = string.split(attr(e, "depends-on"))
 210         self.upgraded = True
 211
 212         self.parse_xml(e)
 213
 214         self.__topdir = None
 215
 216     def get_topdir(self):
 217         if not self.__topdir:
 218             self.__topdir = tempfile.mkdtemp(prefix='B.', dir='/tmp')
 219         return self.__topdir
 220
 221     def parse_xml(self, e):
 222         for c in e.childNodes:
 223             if is_blank(c): continue
 224
 225             if c.nodeType != Element.ELEMENT_NODE:
 226                 log.panic("xml: evil batch child %d" % c.nodeType)
 227             if c.nodeName == "src-rpm":
 228                 self.src_rpm = text(c)
 229             elif c.nodeName == "spec":
 230                 # normalize specname, specname is used as buildlog and we don't
 231                 # want to be exposed to directory traversal attacks
 232                 self.spec = text(c).split('/')[-1]
 233             elif c.nodeName == "command":
 234                 self.spec = "COMMAND"
 235                 self.command = text(c).strip()
 236                 self.command_flags = string.split(attr(c, "flags", ""))
 237             elif c.nodeName == "info":
 238                 self.info = text(c)
 239             elif c.nodeName == "kernel":
 240                 self.kernel = text(c)
 241             elif c.nodeName == "define":
 242                 define = attr(c, "name")
 243                 self.defines[define] = text(c)
 244             elif c.nodeName == "target":
 245                 self.target.append(text(c))
 246             elif c.nodeName == "skip":
 247                 self.skip.append(text(c))
 248             elif c.nodeName == "branch":
 249                 self.branch = text(c)
 250             elif c.nodeName == "builder":
 251                 key = text(c)
 252                 self.builders.append(key)
 253                 self.builders_status[key] = attr(c, "status", "?")
 254                 self.builders_status_time[key] = attr(c, "time", "0")
 255                 self.builders_status_buildtime[key] = "0" #attr(c, "buildtime", "0")
 256             elif c.nodeName == "with":
 257                 self.bconds_with.append(text(c))
 258             elif c.nodeName == "without":
 259                 self.bconds_without.append(text(c))
 260             else:
 261                 log.panic("xml: evil batch child (%s)" % c.nodeName)
 262
 263     def get_package_name(self):
 264         if len(self.spec) <= 5:
 265             return None
 266         return self.spec[:-5]
 267
 268     def tmpdir(self):
 269         """
 270         return tmpdir for this batch job building
 271         """
 272         # it's better to have TMPDIR and BUILD dir on same partition:
 273         # + /usr/bin/bzip2 -dc /home/services/builder/rpm/packages/kernel/patch-2.6.27.61.bz2
 274         # patch: **** Can't rename file /tmp/B.a1b1d3/poKWwRlp to drivers/scsi/hosts.c : No such file or directory
 275         path = os.path.join(self.get_topdir(), 'BUILD', 'tmp')
 276         return path
 277
 278     def is_done(self):
 279         ok = 1
 280         for b in self.builders:
 281             s = self.builders_status[b]
 282             if not s.startswith("OK") and not s.startswith("SKIP") and not s.startswith("UNSUPP") and not s.startswith("FAIL"):
 283                 ok = 0
 284         return ok
 285
 286     def dump(self, f):
 287         f.write("  batch: %s/%s\n" % (self.src_rpm, self.spec))
 288         f.write("    info: %s\n" % self.info)
 289         f.write("    kernel: %s\n" % self.kernel)
 290         f.write("    defines: %s\n" % self.defines_string())
 291         f.write("    target: %s\n" % self.target_string())
 292         f.write("    branch: %s\n" % self.branch)
 293         f.write("    bconds: %s\n" % self.bconds_string())
 294         builders = []
 295         for b in self.builders:
 296             builders.append("%s:%s" % (b, self.builders_status[b]))
 297         f.write("    builders: %s\n" % string.join(builders))
 298
 299     def is_command(self):
 300         return self.command != ""
 301
 302     # return structure usable for json encoding
 303     def dump_json(self):
 304         return dict(
 305             command=self.command,
 306             command_flags=self.command_flags,
 307
 308             spec=self.spec,
 309             branch=self.branch,
 310             package=self.spec[:-5],
 311             src_rpm=self.src_rpm,
 312
 313             bconds_with=self.bconds_with,
 314             bconds_without=self.bconds_without,
 315
 316             kernel=self.kernel,
 317             target=self.target,
 318             defines=self.defines,
 319
 320             builders=self.builders,
 321             builders_status=self.builders_status,
 322             builders_status_time=self.builders_status_time,
 323             builders_status_buildtime=self.builders_status_buildtime,
 324         )
 325
 326     def dump_html(self, f, rid):
 327         f.write("<li>\n")
 328         if self.is_command():
 329             desc = "SH: <pre>%s</pre> flags: [%s]" % (self.command, ' '.join(self.command_flags))
 330         else:
 331             package_url = "http://git.pld-linux.org/gitweb.cgi?p=packages/%(package)s.git;f=%(spec)s;h=%(branch)s;a=shortlog" % {
 332                 'spec': urllib.quote(self.spec),
 333                 'branch': urllib.quote(self.branch),
 334                 'package': urllib.quote(self.spec[:-5]),
 335             }
 336             desc = "%(src_rpm)s (<a href=\"%(package_url)s\">%(spec)s -r %(branch)s</a>%(rpmopts)s)" % {
 337                 'src_rpm': self.src_rpm,
 338                 'spec': self.spec,
 339                 'branch': self.branch,
 340                 'rpmopts': self.bconds_string() + self.kernel_string() + self.target_string() + self.defines_string(),
 341                 'package_url': package_url,
 342             }
 343         f.write("%s <small>[" % desc)
 344         builders = []
 345         for b in self.builders:
 346             s = self.builders_status[b]
 347             if s.startswith("OK"):
 348                 c = "green"
 349             elif s.startswith("FAIL"):
 350                 c = "red"
 351             elif s.startswith("SKIP"):
 352                 c = "blue"
 353             elif s.startswith("UNSUPP"):
 354                 c = "fuchsia"
 355             else:
 356                 c = "black"
 357             link_pre = ""
 358             link_post = ""
 359             if (s.startswith("OK") or s.startswith("SKIP") or s.startswith("UNSUPP") or s.startswith("FAIL")) and len(self.spec) > 5:
 360                 if self.is_command():
 361                     bl_name = "command"
 362                 else:
 363                     bl_name = self.spec[:len(self.spec)-5]
 364                 lin_ar = b.replace('noauto-','')
 365                 path = "/%s/%s/%s,%s.bz2" % (lin_ar.replace('-','/'), s, bl_name, rid)
 366                 is_ok = 0
 367                 if s.startswith("OK"):
 368                     is_ok = 1
 369                 bld = lin_ar.split('-')
 370                 tree_name = '-'.join(bld[:-1])
 371                 tree_arch = '-'.join(bld[-1:])
 372                 link_pre = "<a href=\"%s/index.php?dist=%s&arch=%s&ok=%d&name=%s&id=%s&action=tail\">" \
 373                         % (config.buildlogs, urllib.quote(tree_name), urllib.quote(tree_arch), is_ok, urllib.quote(bl_name), urllib.quote(rid))
 374                 link_post = "</a>"
 375
 376             def ftime(s):
 377                 t = float(s)
 378                 if t > 0:
 379                     return time.asctime(time.localtime(t))
 380                 else:
 381                     return 'N/A'
 382
 383             tooltip = "last update: %(time)s\nbuild time: %(buildtime)s" % {
 384                 'time' : ftime(self.builders_status_time[b]),
 385                 'buildtime' : ftime(self.builders_status_buildtime[b]),
 386             }
 387             builders.append(link_pre +
 388                 "<font color='%(color)s'><b title=\"%(tooltip)s\">%(builder)s:%(status)s</b></font>" % {
 389                     'color' : c,
 390                     'builder' : b,
 391                     'status' : s,
 392                     'tooltip' : cgi.escape(tooltip, True),
 393             }
 394             + link_post)
 395         f.write("%s]</small></li>\n" % string.join(builders))
 396
 397     def rpmbuild_opts(self):
 398         """
 399             return all rpmbuild options related to this build
 400         """
 401         rpmopts = self.bconds_string() + self.kernel_string() + self.target_string() + self.defines_string()
 402         rpmdefs = \
 403             "--define '_topdir %s' " % self.get_topdir() + \
 404             "--define '_specdir %{_topdir}' "  \
 405             "--define '_sourcedir %{_specdir}' " \
 406             "--define '_rpmdir %{_topdir}/RPMS' " \
 407             "--define '_builddir %{_topdir}/BUILD' "
 408         return rpmdefs + rpmopts
 409
 410     def php_ignores(self, php_version):
 411         # available php versions in distro
 412         php_versions = ['4', '5.2', '5.3', '5.4', '5.5', '5.6', '7.0', '7.1', '7.2', '7.3', '7.4', '8.0']
 413
 414         # remove current php version
 415         try:
 416             php_versions.remove(php_version)
 417         except ValueError:
 418             log.notice("Attempt to remove inexistent key '%s' from %s" % (php_version, php_versions))
 419             pass
 420
 421         # map them to poldek ignores
 422         # always ignore hhvm
 423         res = ['hhvm-*']
 424         for v in list(map(php_ver_to_name, php_versions)):
 425             res.append("php%s-*" % v)
 426
 427         return res
 428
 429     # build ignore package list
 430     # currently only php ignore is filled based on build context
 431     def ignores(self):
 432         ignores = []
 433
 434         # add php version based ignores
 435         if self.defines.has_key('php_suffix'):
 436             # current version if -D php_suffix is present
 437             php_version = php_name_to_ver(self.defines['php_suffix'])
 438         else:
 439             php_version = self.DEFAULT_PHP
 440
 441         ignores.extend(self.php_ignores(php_version))
 442
 443         # return empty string if the list is empty
 444         if len(ignores) == 0:
 445             return ""
 446
 447         def add_ignore(s):
 448             return "--ignore=%s" % s
 449
 450         return " ".join(list(map(add_ignore, ignores)))
 451
 452     def kernel_string(self):
 453         r = ""
 454         if self.kernel != "":
 455             r = " --define 'alt_kernel " + self.kernel + "'"
 456         return r
 457
 458     def target_string(self):
 459         if len(self.target) > 0:
 460             return " --target " + ",".join(self.target)
 461         else:
 462             return ""
 463
 464     def bconds_string(self):
 465         r = ""
 466         for b in self.bconds_with:
 467             r = r + " --with " + b
 468         for b in self.bconds_without:
 469             r = r + " --without " + b
 470         return r
 471
 472     def defines_string(self):
 473         r = ""
 474         for key,value in self.defines.items():
 475             r += " --define '%s %s'" % (key, value)
 476         return r
 477
 478     def defines_xml(self):
 479         r = ""
 480         for key,value in self.defines.items():
 481             r += "<define name='%s'>%s</define>\n" % (escape(key), escape(value))
 482         return r
 483
 484     def default_target(self, arch):
 485         self.target.append("%s-pld-linux" % arch)
 486
 487     def write_to(self, f):
 488         f.write("""
 489          <batch id='%s' depends-on='%s'>
 490            <src-rpm>%s</src-rpm>
 491            <command flags="%s">%s</command>
 492            <spec>%s</spec>
 493            <branch>%s</branch>
 494            <info>%s</info>\n""" % (self.b_id,
 495                  string.join(list(map(lambda (b): b.b_id, self.depends_on))),
 496                  escape(self.src_rpm),
 497                  escape(' '.join(self.command_flags)), escape(self.command),
 498                  escape(self.spec), escape(self.branch), escape(self.info)))
 499         if self.kernel != "":
 500             f.write("           <kernel>%s</kernel>\n" % escape(self.kernel))
 501         for b in self.bconds_with:
 502             f.write("           <with>%s</with>\n" % escape(b))
 503         for b in self.target:
 504             f.write("           <target>%s</target>\n" % escape(b))
 505         for b in self.bconds_without:
 506             f.write("           <without>%s</without>\n" % escape(b))
 507         if self.defines:
 508             f.write("           %s\n" % self.defines_xml())
 509         for b in self.builders:
 510             if self.builders_status_buildtime.has_key(b):
 511                 t = self.builders_status_buildtime[b]
 512             else:
 513                 t = "0"
 514             f.write("           <builder status='%s' time='%s' buildtime='%s'>%s</builder>\n" % \
 515                     (escape(self.builders_status[b]), self.builders_status_time[b], t, escape(b)))
 516         f.write("         </batch>\n")
 517
 518     def log_line(self, l):
 519         log.notice(l)
 520         if self.logfile != None:
 521             util.append_to(self.logfile, l)
 522
 523     def expand_builders(batch, all_builders):
 524         all = []
 525         for bld in batch.builders:
 526             res = []
 527             for my_bld in all_builders:
 528                 if fnmatch.fnmatch(my_bld, bld):
 529                     res.append(my_bld)
 530             if res != []:
 531                 all.extend(res)
 532             else:
 533                 all.append(bld)
 534         batch.builders = all
 535
 536 class Notification:
 537     def __init__(self, e):
 538         self.batches = []
 539         self.kind = 'notification'
 540         self.group_id = attr(e, "group-id")
 541         self.builder = attr(e, "builder")
 542         self.batches = {}
 543         self.batches_buildtime = {}
 544         for c in e.childNodes:
 545             if is_blank(c): continue
 546             if c.nodeType != Element.ELEMENT_NODE:
 547                 log.panic("xml: evil notification child %d" % c.nodeType)
 548             if c.nodeName == "batch":
 549                 id = attr(c, "id")
 550                 status = attr(c, "status")
 551                 buildtime = attr(c, "buildtime", "0")
 552                 if not status.startswith("OK") and not status.startswith("SKIP") and not status.startswith("UNSUPP") and not status.startswith("FAIL"):
 553                     log.panic("xml notification: bad status: %s" % status)
 554                 self.batches[id] = status
 555                 self.batches_buildtime[id] = buildtime
 556             else:
 557                 log.panic("xml: evil notification child (%s)" % c.nodeName)
 558
 559     # return structure usable for json encoding
 560     def dump_json(self):
 561         return dict(
 562             id=self.group_id,
 563             builder=self.builder,
 564             batches=self.batches,
 565             batches_buildtime=self.batches_buildtime,
 566         )
 567
 568     def apply_to(self, q):
 569         for r in q.requests:
 570             if r.kind == "group":
 571                 for b in r.batches:
 572                     if self.batches.has_key(b.b_id):
 573                         b.builders_status[self.builder] = self.batches[b.b_id]
 574                         b.builders_status_time[self.builder] = time.time()
 575                         b.builders_status_buildtime[self.builder] = "0" #self.batches_buildtime[b.b_id]
 576
 577 def build_request(e):
 578     if e.nodeType != Element.ELEMENT_NODE:
 579         log.panic("xml: evil request element")
 580     if e.nodeName == "group":
 581         return Group(e)
 582     elif e.nodeName == "notification":
 583         return Notification(e)
 584     elif e.nodeName == "command":
 585         # FIXME
 586         return Command(e)
 587     else:
 588         log.panic("xml: evil request [%s]" % e.nodeName)
 589
 590 def parse_request(f):
 591     d = parseString(f)
 592     return build_request(d.documentElement)
 593
 594 def parse_requests(f):
 595     d = parseString(f)
 596     res = []
 597     for r in d.documentElement.childNodes:
 598         if is_blank(r): continue
 599         res.append(build_request(r))
 600     return res