make global functions global

[projects/pld-builder.new.git] / PLD_Builder / request.py

# vi: encoding=utf-8 ts=8 sts=4 sw=4 et

from xml.dom.minidom import *
from datetime import datetime
import string
import time
import xml.sax.saxutils
import fnmatch
import os
import urllib
import cgi
import pytz
import tempfile

import util
import log
from acl import acl
from config import config

__all__ = ['parse_request', 'parse_requests']

def text(e):
    res = ""
    for n in e.childNodes:
        if n.nodeType != Element.TEXT_NODE:
            log.panic("xml: text expected in <%s>, got %d" % (e.nodeName, n.nodeType))
        res += n.nodeValue
    return res

def attr(e, a, default = None):
    try:
        return e.attributes[a].value
    except:
        if default != None:
            return default
        raise

def escape(s):
    return xml.sax.saxutils.escape(s)

# return timestamp with timezone information
# so we could parse it in javascript
def tzdate(t):
    # as strftime %z is unofficial, and does not work, need to make it numeric ourselves
    date = time.strftime("%a %b %d %Y %H:%M:%S", time.localtime(t))
    # NOTE: the altzone is showing CURRENT timezone, not what the "t" reflects
    # NOTE: when DST is off timezone gets it right, altzone not
    if time.daylight:
        tzoffset = time.altzone
    else:
        tzoffset = time.timezone
    tz = '%+05d' % (-tzoffset / 3600 * 100)
    return date + ' ' + tz

# return date in iso8601 format
def iso8601(ts, timezone='UTC'):
    tz = pytz.timezone(timezone)
    dt = datetime.fromtimestamp(ts, tz)
    return dt.isoformat()

def is_blank(e):
    return e.nodeType == Element.TEXT_NODE and string.strip(e.nodeValue) == ""

class Group:
    def __init__(self, e):
        self.batches = []
        self.kind = 'group'
        self.id = attr(e, "id")
        self.no = int(attr(e, "no"))
        self.priority = 2
        self.time = time.time()
        self.requester = ""
        self.max_jobs = 0
        self.requester_email = ""
        self.flags = string.split(attr(e, "flags", ""))
        for c in e.childNodes:
            if is_blank(c): continue

            if c.nodeType != Element.ELEMENT_NODE:
                log.panic("xml: evil group child %d" % c.nodeType)
            if c.nodeName == "batch":
                self.batches.append(Batch(c))
            elif c.nodeName == "requester":
                self.requester = text(c)
                self.requester_email = attr(c, "email", "")
            elif c.nodeName == "priority":
                self.priority = int(text(c))
            elif c.nodeName == "time":
                self.time = int(text(c))
            elif c.nodeName == "maxjobs":
                self.max_jobs = int(text(c))
            else:
                log.panic("xml: evil group child (%s)" % c.nodeName)
        # note that we also check that group is sorted WRT deps
        m = {}
        for b in self.batches:
            deps = []
            m[b.b_id] = b
            for dep in b.depends_on:
                if m.has_key(dep):
                    # avoid self-deps
                    if id(m[dep]) != id(b):
                        deps.append(m[dep])
                else:
                    log.panic("xml: dependency not found in group")
            b.depends_on = deps
        if self.requester_email == "" and self.requester != "":
            self.requester_email = acl.user(self.requester).mail_to()

    def dump(self, f):
        f.write("group: %d (id=%s pri=%d)\n" % (self.no, self.id, self.priority))
        f.write("  from: %s\n" % self.requester)
        f.write("  flags: %s\n" % string.join(self.flags))
        f.write("  time: %s\n" % time.asctime(time.localtime(self.time)))
        for b in self.batches:
            b.dump(f)
        f.write("\n")

    # return structure usable for json encoding
    def dump_json(self):
        batches = []
        for b in self.batches:
            batches.append(b.dump_json())

        return dict(
            no=self.no,
            id=self.id,
            time=self.time,
            requester=self.requester,
            priority=self.priority,
            max_jobs=self.max_jobs,
            flags=self.flags,
            batches=batches,
        )

    def dump_html(self, f):
        f.write(
            "<div id=\"%(no)d\" class=\"request %(flags)s\">\n"
            "<a href=\"#%(no)d\">%(no)d</a>. "
            "<time class=\"timeago\" datetime=\"%(datetime)s\">%(time)s</time> "
            "from <b class=requester>%(requester)s</b> "
            "<small>%(id)s, prio=%(priority)d, jobs=%(max_jobs)d, %(flags)s</small>\n"
        % {
            'no': self.no,
            'id': '<a href="srpms/%(id)s">%(id)s</a>' % {'id': self.id},
            'time': escape(tzdate(self.time)),
            'datetime': escape(iso8601(self.time)),
            'requester': escape(self.requester),
            'priority': self.priority,
            'max_jobs': self.max_jobs,
            'flags': string.join(self.flags)
        })
        f.write("<ol>\n")
        for b in self.batches:
            b.dump_html(f, self.id)
        f.write("</ol>\n")
        f.write("</div>\n")

    def write_to(self, f):
        f.write("""
       <group id="%s" no="%d" flags="%s">
         <requester email='%s'>%s</requester>
         <time>%d</time>
         <priority>%d</priority>
         <maxjobs>%d</maxjobs>\n""" % (self.id, self.no, string.join(self.flags),
                    escape(self.requester_email), escape(self.requester),
                    self.time, self.priority, self.max_jobs))
        for b in self.batches:
            b.write_to(f)
        f.write("       </group>\n\n")

    def is_done(self):
        ok = 1
        for b in self.batches:
            if not b.is_done():
                ok = 0
        return ok

# transform php package name (52) to version (5.2)
def php_name_to_ver(v):
    return '.'.join(list(v))

# transform php version (5.2) to package name (52)
def php_ver_to_name(v):
    return v.replace('.', '')

class Batch:
    DEFAULT_PHP = '5.3'

    def __init__(self, e):
        self.bconds_with = []
        self.bconds_without = []
        self.builders = []
        self.builders_status = {}
        self.builders_status_time = {}
        self.builders_status_buildtime = {}
        self.kernel = ""
        self.defines = {}
        self.target = []
        self.branch = ""
        self.src_rpm = ""
        self.info = ""
        self.spec = ""
        self.command = ""
        self.command_flags = []
        self.skip = []
        self.gb_id = ""
        self.b_id = attr(e, "id")
        self.depends_on = string.split(attr(e, "depends-on"))
        self.upgraded = True

        self.parse_xml(e)

        self.__topdir = None

    def get_topdir(self):
        if not self.__topdir:
            self.__topdir = tempfile.mkdtemp(prefix='B.', dir='/tmp')
        return self.__topdir

    def parse_xml(self, e):
        for c in e.childNodes:
            if is_blank(c): continue

            if c.nodeType != Element.ELEMENT_NODE:
                log.panic("xml: evil batch child %d" % c.nodeType)
            if c.nodeName == "src-rpm":
                self.src_rpm = text(c)
            elif c.nodeName == "spec":
                # normalize specname, specname is used as buildlog and we don't
                # want to be exposed to directory traversal attacks
                self.spec = text(c).split('/')[-1]
            elif c.nodeName == "command":
                self.spec = "COMMAND"
                self.command = text(c).strip()
                self.command_flags = string.split(attr(c, "flags", ""))
            elif c.nodeName == "info":
                self.info = text(c)
            elif c.nodeName == "kernel":
                self.kernel = text(c)
            elif c.nodeName == "define":
                define = attr(c, "name")
                self.defines[define] = text(c)
            elif c.nodeName == "target":
                self.target.append(text(c))
            elif c.nodeName == "skip":
                self.skip.append(text(c))
            elif c.nodeName == "branch":
                self.branch = text(c)
            elif c.nodeName == "builder":
                key = text(c)
                self.builders.append(key)
                self.builders_status[key] = attr(c, "status", "?")
                self.builders_status_time[key] = attr(c, "time", "0")
                self.builders_status_buildtime[key] = "0" #attr(c, "buildtime", "0")
            elif c.nodeName == "with":
                self.bconds_with.append(text(c))
            elif c.nodeName == "without":
                self.bconds_without.append(text(c))
            else:
                log.panic("xml: evil batch child (%s)" % c.nodeName)

    def get_package_name(self):
        if len(self.spec) <= 5:
            return None
        return self.spec[:-5]

    def tmpdir(self):
        """
        return tmpdir for this batch job building
        """
        # it's better to have TMPDIR and BUILD dir on same partition:
        # + /usr/bin/bzip2 -dc /home/services/builder/rpm/packages/kernel/patch-2.6.27.61.bz2
        # patch: **** Can't rename file /tmp/B.a1b1d3/poKWwRlp to drivers/scsi/hosts.c : No such file or directory
        path = os.path.join(self.get_topdir(), 'BUILD', 'tmp')
        return path

    def is_done(self):
        ok = 1
        for b in self.builders:
            s = self.builders_status[b]
            if not s.startswith("OK") and not s.startswith("SKIP") and not s.startswith("UNSUPP") and not s.startswith("FAIL"):
                ok = 0
        return ok

    def dump(self, f):
        f.write("  batch: %s/%s\n" % (self.src_rpm, self.spec))
        f.write("    info: %s\n" % self.info)
        f.write("    kernel: %s\n" % self.kernel)
        f.write("    defines: %s\n" % self.defines_string())
        f.write("    target: %s\n" % self.target_string())
        f.write("    branch: %s\n" % self.branch)
        f.write("    bconds: %s\n" % self.bconds_string())
        builders = []
        for b in self.builders:
            builders.append("%s:%s" % (b, self.builders_status[b]))
        f.write("    builders: %s\n" % string.join(builders))

    def is_command(self):
        return self.command != ""

    # return structure usable for json encoding
    def dump_json(self):
        return dict(
            command=self.command,
            command_flags=self.command_flags,

            spec=self.spec,
            branch=self.branch,
            package=self.spec[:-5],
            src_rpm=self.src_rpm,

            bconds_with=self.bconds_with,
            bconds_without=self.bconds_without,

            kernel=self.kernel,
            target=self.target,
            defines=self.defines,

            builders=self.builders,
            builders_status=self.builders_status,
            builders_status_time=self.builders_status_time,
            builders_status_buildtime=self.builders_status_buildtime,
        )

    def dump_html(self, f, rid):
        f.write("<li>\n")
        if self.is_command():
            desc = "SH: <pre>%s</pre> flags: [%s]" % (self.command, ' '.join(self.command_flags))
        else:
            package_url = "http://git.pld-linux.org/gitweb.cgi?p=packages/%(package)s.git;f=%(spec)s;h=%(branch)s;a=shortlog" % {
                'spec': urllib.quote(self.spec),
                'branch': urllib.quote(self.branch),
                'package': urllib.quote(self.spec[:-5]),
            }
            desc = "%(src_rpm)s (<a href=\"%(package_url)s\">%(spec)s -r %(branch)s</a>%(rpmopts)s)" % {
                'src_rpm': self.src_rpm,
                'spec': self.spec,
                'branch': self.branch,
                'rpmopts': self.bconds_string() + self.kernel_string() + self.target_string() + self.defines_string(),
                'package_url': package_url,
            }
        f.write("%s <small>[" % desc)
        builders = []
        for b in self.builders:
            s = self.builders_status[b]
            if s.startswith("OK"):
                c = "green"
            elif s.startswith("FAIL"):
                c = "red"
            elif s.startswith("SKIP"):
                c = "blue"
            elif s.startswith("UNSUPP"):
                c = "fuchsia"
            else:
                c = "black"
            link_pre = ""
            link_post = ""
            if (s.startswith("OK") or s.startswith("SKIP") or s.startswith("UNSUPP") or s.startswith("FAIL")) and len(self.spec) > 5:
                if self.is_command():
                    bl_name = "command"
                else:
                    bl_name = self.spec[:len(self.spec)-5]
                lin_ar = b.replace('noauto-','')
                path = "/%s/%s/%s,%s.bz2" % (lin_ar.replace('-','/'), s, bl_name, rid)
                is_ok = 0
                if s.startswith("OK"):
                    is_ok = 1
                bld = lin_ar.split('-')
                tree_name = '-'.join(bld[:-1])
                tree_arch = '-'.join(bld[-1:])
                link_pre = "<a href=\"%s/index.php?dist=%s&arch=%s&ok=%d&name=%s&id=%s&action=tail\">" \
                        % (config.buildlogs, urllib.quote(tree_name), urllib.quote(tree_arch), is_ok, urllib.quote(bl_name), urllib.quote(rid))
                link_post = "</a>"

            def ftime(s):
                t = float(s)
                if t > 0:
                    return time.asctime(time.localtime(t))
                else:
                    return 'N/A'

            tooltip = "last update: %(time)s\nbuild time: %(buildtime)s" % {
                'time' : ftime(self.builders_status_time[b]),
                'buildtime' : ftime(self.builders_status_buildtime[b]),
            }
            builders.append(link_pre +
                "<font color='%(color)s'><b title=\"%(tooltip)s\">%(builder)s:%(status)s</b></font>" % {
                    'color' : c,
                    'builder' : b,
                    'status' : s,
                    'tooltip' : cgi.escape(tooltip, True),
            }
            + link_post)
        f.write("%s]</small></li>\n" % string.join(builders))

    def rpmbuild_opts(self):
        """
            return all rpmbuild options related to this build
        """
        rpmopts = self.bconds_string() + self.kernel_string() + self.target_string() + self.defines_string()
        rpmdefs = \
            "--define '_topdir %s' " % self.get_topdir() + \
            "--define '_specdir %{_topdir}' "  \
            "--define '_sourcedir %{_specdir}' " \
            "--define '_rpmdir %{_topdir}/RPMS' " \
            "--define '_builddir %{_topdir}/BUILD' "
        return rpmdefs + rpmopts

    def php_ignores(self, php_version):
        # available php versions in distro
        php_versions = ['4', '5.2', '5.3', '5.4', '5.5', '5.6', '7.0', '7.1', '7.2']

        # remove current php version
        try:
            php_versions.remove(php_version)
        except ValueError:
            log.notice("Attempt to remove inexistent key '%s' from %s" % (php_version, php_versions))
            pass

        # map them to poldek ignores
        # always ignore hhvm
        res = ['hhvm-*']
        for v in map(php_ver_to_name, php_versions):
            res.append("php%s-*" % v)

        return res

    # build ignore package list
    # currently only php ignore is filled based on build context
    def ignores(self):
        ignores = []

        # add php version based ignores
        if self.defines.has_key('php_suffix'):
            # current version if -D php_suffix is present
            php_version = php_name_to_ver(self.defines['php_suffix'])
        else:
            php_version = self.DEFAULT_PHP

        ignores.extend(self.php_ignores(php_version))

        # return empty string if the list is empty
        if len(ignores) == 0:
            return ""

        def add_ignore(s):
            return "--ignore=%s" % s

        return " ".join(map(add_ignore, ignores))

    def kernel_string(self):
        r = ""
        if self.kernel != "":
            r = " --define 'alt_kernel " + self.kernel + "'"
        return r

    def target_string(self):
        if len(self.target) > 0:
            return " --target " + ",".join(self.target)
        else:
            return ""

    def bconds_string(self):
        r = ""
        for b in self.bconds_with:
            r = r + " --with " + b
        for b in self.bconds_without:
            r = r + " --without " + b
        return r

    def defines_string(self):
        r = ""
        for key,value in self.defines.items():
            r += " --define '%s %s'" % (key, value)
        return r

    def defines_xml(self):
        r = ""
        for key,value in self.defines.items():
            r += "<define name='%s'>%s</define>\n" % (escape(key), escape(value))
        return r

    def default_target(self, arch):
        self.target.append("%s-pld-linux" % arch)

    def write_to(self, f):
        f.write("""
         <batch id='%s' depends-on='%s'>
           <src-rpm>%s</src-rpm>
           <command flags="%s">%s</command>
           <spec>%s</spec>
           <branch>%s</branch>
           <info>%s</info>\n""" % (self.b_id,
                 string.join(map(lambda (b): b.b_id, self.depends_on)),
                 escape(self.src_rpm),
                 escape(' '.join(self.command_flags)), escape(self.command),
                 escape(self.spec), escape(self.branch), escape(self.info)))
        if self.kernel != "":
            f.write("           <kernel>%s</kernel>\n" % escape(self.kernel))
        for b in self.bconds_with:
            f.write("           <with>%s</with>\n" % escape(b))
        for b in self.target:
            f.write("           <target>%s</target>\n" % escape(b))
        for b in self.bconds_without:
            f.write("           <without>%s</without>\n" % escape(b))
        if self.defines:
            f.write("           %s\n" % self.defines_xml())
        for b in self.builders:
            if self.builders_status_buildtime.has_key(b):
                t = self.builders_status_buildtime[b]
            else:
                t = "0"
            f.write("           <builder status='%s' time='%s' buildtime='%s'>%s</builder>\n" % \
                    (escape(self.builders_status[b]), self.builders_status_time[b], t, escape(b)))
        f.write("         </batch>\n")

    def log_line(self, l):
        log.notice(l)
        if self.logfile != None:
            util.append_to(self.logfile, l)

    def expand_builders(batch, all_builders):
        all = []
        for bld in batch.builders:
            res = []
            for my_bld in all_builders:
                if fnmatch.fnmatch(my_bld, bld):
                    res.append(my_bld)
            if res != []:
                all.extend(res)
            else:
                all.append(bld)
        batch.builders = all

class Notification:
    def __init__(self, e):
        self.batches = []
        self.kind = 'notification'
        self.group_id = attr(e, "group-id")
        self.builder = attr(e, "builder")
        self.batches = {}
        self.batches_buildtime = {}
        for c in e.childNodes:
            if is_blank(c): continue
            if c.nodeType != Element.ELEMENT_NODE:
                log.panic("xml: evil notification child %d" % c.nodeType)
            if c.nodeName == "batch":
                id = attr(c, "id")
                status = attr(c, "status")
                buildtime = attr(c, "buildtime", "0")
                if not status.startswith("OK") and not status.startswith("SKIP") and not status.startswith("UNSUPP") and not status.startswith("FAIL"):
                    log.panic("xml notification: bad status: %s" % status)
                self.batches[id] = status
                self.batches_buildtime[id] = buildtime
            else:
                log.panic("xml: evil notification child (%s)" % c.nodeName)

    # return structure usable for json encoding
    def dump_json(self):
        return dict(
            id=self.group_id,
            builder=self.builder,
            batches=self.batches,
            batches_buildtime=self.batches_buildtime,
        )

    def apply_to(self, q):
        for r in q.requests:
            if r.kind == "group":
                for b in r.batches:
                    if self.batches.has_key(b.b_id):
                        b.builders_status[self.builder] = self.batches[b.b_id]
                        b.builders_status_time[self.builder] = time.time()
                        b.builders_status_buildtime[self.builder] = "0" #self.batches_buildtime[b.b_id]

def build_request(e):
    if e.nodeType != Element.ELEMENT_NODE:
        log.panic("xml: evil request element")
    if e.nodeName == "group":
        return Group(e)
    elif e.nodeName == "notification":
        return Notification(e)
    elif e.nodeName == "command":
        # FIXME
        return Command(e)
    else:
        log.panic("xml: evil request [%s]" % e.nodeName)

def parse_request(f):
    d = parseString(f)
    return build_request(d.documentElement)

def parse_requests(f):
    d = parseString(f)
    res = []
    for r in d.documentElement.childNodes:
        if is_blank(r): continue
        res.append(build_request(r))
    return res