]> git.pld-linux.org Git - projects/pld-builder.new.git/commitdiff
Merge changes from PLD CVS
authorKacper Kornet <draenog@pld-linux.org>
Tue, 10 Jul 2012 04:16:44 +0000 (05:16 +0100)
committerKacper Kornet <draenog@pld-linux.org>
Tue, 10 Jul 2012 04:16:44 +0000 (05:16 +0100)
PLD_Builder/file_sender.py
PLD_Builder/install.py
PLD_Builder/request.py
PLD_Builder/rpm_builder.py
client/make-request.sh
client/rebuild-th-kernel.sh [new file with mode: 0644]
client/send-last-th-tag.sh [new file with mode: 0644]
etc/crontab

index b9e89908bd14315a7477ff09bfedf1ea1a2a1994..f8d41f377cea9429a50dac8197dfc17b0e11f269 100644 (file)
@@ -70,7 +70,7 @@ def rsync_file(src, target, host):
     # password, it opens /dev/tty and still asks if password is needed and
     # missing, therefore we always set RSYNC_PASSWORD env var
     os.environ["RSYNC_PASSWORD"] = password
-    rsync = "rsync --verbose --archive"
+    rsync = "rsync --verbose --archive --timeout=360 --contimeout=360"
     f = os.popen("%s %s %s 2>&1" % (rsync, src, target))
     p = f.read()
     ret = f.close()
@@ -81,7 +81,7 @@ def rsync_file(src, target, host):
 
 def rsync_ssh_file(src, target):
     global problems
-    rsync = "rsync --verbose --archive -e ssh"
+    rsync = "rsync --verbose --archive --timeout=360 -e ssh"
     f = os.popen("%s %s %s 2>&1 < /dev/null" % (rsync, src, target))
     p = f.read()
     ret = f.close()
index 3e23699ab217b945109e28d04874b033073f3cfc..6915133eaf58373e7e10891671bfb463d2f8c63e 100644 (file)
@@ -1,6 +1,6 @@
 # vi: encoding=utf-8 ts=8 sts=4 sw=4 et
 
-import re
+import re, os
 import string
 import StringIO
 
@@ -100,10 +100,14 @@ def uninstall(conflicting, b):
 
 def uninstall_self_conflict(b):
     b.log_line("checking BuildConflict-ing packages")
-    rpmbuild_opt = "%s %s %s" % (b.target_string(), b.kernel_string(), b.bconds_string())
-    tmpdir = "/tmp/BR." + b.b_id[0:6]
-    f = chroot.popen("cd rpm/SPECS; TMPDIR=%s rpmbuild -bp --nobuild --short-circuit --define 'prep exit 0' %s %s 2>&1" \
-            % (tmpdir, rpmbuild_opt, b.spec))
+    packagename = b.spec[:-5]
+    tmpdir = os.environ.get('HOME') + "/rpm/BUILD/%s/tmp" % packagename
+    f = chroot.popen("set -ex; TMPDIR=%(tmpdir)s rpmbuild -bp --nobuild --short-circuit --define 'prep exit 0' %(rpmdefs)s rpm/packages/%(package)s/%(spec)s 2>&1" % {
+        'tmpdir': tmpdir,
+        'rpmdefs' : b.rpmbuild_opts(),
+        'package' : packagename,
+        'spec': b.spec,
+    })
     # java-sun >= 1.5 conflicts with soprano-2.1.67-1.src
     # java-sun conflicts with soprano-2.1.67-1.src
     rx = re.compile(r"\s+(?P<name>[\w-]+)\s+.*conflicts with [^\s]+-[^-]+-[^-]+\.src($| .*)")
@@ -124,10 +128,15 @@ def install_br(r, b):
         # ignore internal rpm dependencies, see lib/rpmns.c for list
         ignore_br = re.compile(r'^\s*(rpmlib|cpuinfo|getconf|uname|soname|user|group|mounted|diskspace|digest|gnupg|macro|envvar|running|sanitycheck|vcheck|signature|verify|exists|executable|readable|writable)\(.*')
 
-        tmpdir = "/tmp/BR." + b.b_id[0:6]
+        packagename = b.spec[:-5]
+        tmpdir = os.environ.get('HOME') + "/rpm/BUILD/%s/tmp" % packagename
         chroot.run("install -m 700 -d %s" % tmpdir)
-        cmd = "cd rpm/SPECS; TMPDIR=%s rpmbuild --nobuild %s %s 2>&1" \
-                    % (tmpdir, b.bconds_string(), b.spec)
+        cmd = "set -ex; TMPDIR=%(tmpdir)s rpmbuild --nobuild %(rpmdefs)s rpm/packages/%(package)s/%(spec)s 2>&1" % {
+            'tmpdir': tmpdir,
+            'rpmdefs' : b.rpmbuild_opts(),
+            'package' : packagename,
+            'spec': b.spec,
+        }
         f = chroot.popen(cmd)
         rx = re.compile(r"^\s*(?P<name>[^\s]+) .*is needed by")
         needed = {}
@@ -138,7 +147,6 @@ def install_br(r, b):
             if m and not ignore_br.match(l):
                 needed[m.group('name')] = 1
         f.close()
-        chroot.run("rm -rf %s" % tmpdir)
         return needed
 
     needed = get_missing_br(r, b);
index 7b779ac3b450d9c484ab2470127cb9ed8e19304b..c87d644defef8159036158926c41931b4b03a312 100644 (file)
@@ -235,7 +235,7 @@ class Batch:
         if self.is_command():
             desc = "SH: <pre>%s</pre> flags: [%s]" % (self.command, ' '.join(self.command_flags))
         else:
-            package_url = "http://cvs.pld-linux.org/cgi-bin/cvsweb.cgi/packages/%(package)s/%(spec)s?only_with_tag=%(branch)s" % {
+            package_url = "http://cvs.pld-linux.org/packages/%(package)s/%(spec)s?only_with_tag=%(branch)s" % {
                 'spec': self.spec,
                 'branch': self.branch,
                 'package': self.spec[:-5],
@@ -301,6 +301,18 @@ class Batch:
             + link_post)
         f.write("%s]</small></li>\n" % string.join(builders))
 
+    def rpmbuild_opts(self):
+        """
+            return all rpmbuild options related to this build
+        """
+        bconds = self.bconds_string() + self.kernel_string() + self.target_string()
+        rpmdefs = \
+            "--define '_topdir %(echo $HOME/rpm)' " \
+            "--define '_specdir %{_topdir}/packages/%{name}' "  \
+            "--define '_sourcedir %{_specdir}' " \
+            "--define '_builddir %{_topdir}/BUILD/%{name}' "
+        return rpmdefs + bconds
+
     def kernel_string(self):
         r = ""
         if self.kernel != "":
@@ -313,9 +325,6 @@ class Batch:
         else:
             return ""
 
-    def default_target(self, arch):
-        self.target.append("%s-pld-linux" % arch)
-
     def bconds_string(self):
         r = ""
         for b in self.bconds_with:
@@ -324,6 +333,9 @@ class Batch:
             r = r + " --without " + b
         return r
 
+    def default_target(self, arch):
+        self.target.append("%s-pld-linux" % arch)
+
     def write_to(self, f):
         f.write("""
          <batch id='%s' depends-on='%s'>
index 66720e2ad40b30b379b016da62157f5ce123e8f7..3072f4d7f35561130cb460e4672b3fadc50e12af 100644 (file)
@@ -144,7 +144,14 @@ def prepare_env():
     """, 'root')
 
 def build_rpm(r, b):
-    status.push("building %s" % b.spec)
+    if len(b.spec) <= 5:
+        # should not really get here
+        b.log_line("error: No .spec not given of malformed: '%s'" % b.spec)
+        res = "FAIL_INTERNAL"
+        return res
+
+    packagename = b.spec[:-5]
+    status.push("building %s (%s)" % (b.spec, packagename))
     b.log_line("request from: %s" % r.requester)
 
     if check_skip_build(r, b):
@@ -156,12 +163,23 @@ def build_rpm(r, b):
     fetch_src(r, b)
     b.log_line("installing srpm: %s" % b.src_rpm)
     res = chroot.run("""
-        install -d rpm/SPECS rpm/SOURCES
-        rpm -Uhv %s
-    """ % b.src_rpm, logfile = b.logfile)
-    chroot.run("rm -f %s" % b.src_rpm, logfile = b.logfile)
+        # b.id %(bid)s
+        set -ex;
+        install -d rpm/packages/%(package)s rpm/BUILD/%(package)s;
+        rpm -Uhv %(rpmdefs)s %(src_rpm)s;
+        rm -f %(src_rpm)s;
+    """ % {
+        'bid' : b.b_id,
+        'package' : packagename,
+        'rpmdefs' : b.rpmbuild_opts(),
+        'src_rpm' : b.src_rpm
+    }, logfile = b.logfile)
     b.files = []
-    tmpdir = "/tmp/B." + b.b_id[0:6]
+
+    # it's better to have TMPDIR and BUILD dir on same partition:
+    # + /usr/bin/bzip2 -dc /home/services/builder/rpm/packages/kernel/patch-2.6.27.61.bz2
+    # patch: **** Can't rename file /tmp/B.a1b1d3/poKWwRlp to drivers/scsi/hosts.c : No such file or directory
+    tmpdir = os.environ.get('HOME') + "/rpm/BUILD/%s/tmp" % packagename
     if res:
         b.log_line("error: installing src rpm failed")
         res = "FAIL_SRPM_INSTALL"
@@ -170,10 +188,15 @@ def build_rpm(r, b):
         chroot.run("install -m 700 -d %s" % tmpdir)
 
         b.default_target(config.arch)
-        rpmbuild_opt = "%s %s %s" % (b.target_string(), b.kernel_string(), b.bconds_string())
         # check for build arch before filling BR
-        cmd = "cd rpm/SPECS; TMPDIR=%s exec nice -n %s rpmbuild -bp --short-circuit --nodeps --define 'prep exit 0' %s %s" % \
-            (tmpdir, config.nice, rpmbuild_opt, b.spec)
+        cmd = "set -ex; TMPDIR=%(tmpdir)s exec nice -n %(nice)s " \
+            "rpmbuild -bp --short-circuit --nodeps %(rpmdefs)s --define 'prep exit 0' rpm/packages/%(package)s/%(spec)s" % {
+            'tmpdir': tmpdir,
+            'nice' : config.nice,
+            'rpmdefs' : b.rpmbuild_opts(),
+            'package' : packagename,
+            'spec': b.spec,
+        }
         res = chroot.run(cmd, logfile = b.logfile)
         if res:
             res = "UNSUPP"
@@ -188,8 +211,16 @@ def build_rpm(r, b):
                 max_jobs = max(min(int(os.sysconf('SC_NPROCESSORS_ONLN') + 1), config.max_jobs), 1)
                 if r.max_jobs > 0:
                     max_jobs = max(min(config.max_jobs, r.max_jobs), 1)
-                cmd = "echo build-id: %s; cd rpm/SPECS; TMPDIR=%s exec nice -n %s rpmbuild -bb --define '_smp_mflags -j%d' %s %s" % \
-                            (r.id, tmpdir, config.nice, max_jobs, rpmbuild_opt, b.spec)
+                cmd = "set -ex; : build-id: %(r_id)s; TMPDIR=%(tmpdir)s exec nice -n %(nice)s " \
+                    "rpmbuild -bb --define '_smp_mflags -j%(max_jobs)d' %(rpmdefs)s rpm/packages/%(package)s/%(spec)s" % {
+                    'r_id' : r.id,
+                    'tmpdir': tmpdir,
+                    'nice' : config.nice,
+                    'rpmdefs' : b.rpmbuild_opts(),
+                    'package' : packagename,
+                    'max_jobs' : max_jobs,
+                    'spec': b.spec,
+                }
                 b.log_line("building RPM using: %s" % cmd)
                 begin_time = time.time()
                 res = chroot.run(cmd, logfile = b.logfile)
@@ -209,10 +240,14 @@ def build_rpm(r, b):
                         res = "FAIL_%s" % last_section.upper()
                 b.files = files
 
-    chroot.run("rm -rf %s; cd rpm/SPECS; rpmbuild --nodeps --nobuild " \
-                         "--clean --rmspec --rmsource %s" % \
-                         (tmpdir, b.spec), logfile = b.logfile)
-    chroot.run("chmod -R u+rwX rpm/BUILD/*; rm -rf rpm/BUILD/*", logfile = b.logfile)
+    chroot.run("""
+        set -ex;
+        rpmbuild %(rpmdefs)s --nodeps --nobuild --clean --rmspec --rmsource rpm/packages/%(package)s/%(spec)s
+        rm -rf %(tmpdir)s;
+        chmod -R u+rwX rpm/BUILD/%(package)s;
+        rm -rf rpm/BUILD/%(package)s;
+    """ %
+        {'tmpdir' : tmpdir, 'spec': b.spec, 'package' : packagename, 'rpmdefs' : b.rpmbuild_opts()}, logfile = b.logfile)
 
     def ll(l):
         util.append_to(b.logfile, l)
index b79854b2a9215db2ee6d8856f795996e7b85121e..a213be8b6f8767c353ff0b86e4d1ed6755c50c51 100755 (executable)
@@ -216,11 +216,35 @@ EOF
        exit 0
 }
 
+# validate distro, set $distro
+set_distro() {
+       case "$1" in
+       ac)
+               ;;
+       ac-java|ac-xen)
+               ;;
+       ti)
+               ;;
+       ti-dev)
+               ;;
+       th)
+               ;;
+       th-java)
+               ;;
+       aidath)
+               ;;
+       *)
+               die "distro \`$1' not known"
+               ;;
+       esac
+
+       distro=$1
+}
 
 while [ $# -gt 0 ] ; do
        case "$1" in
                --distro | -d)
-                       distro=$2
+                       set_distro $2
                        shift
                        ;;
 
diff --git a/client/rebuild-th-kernel.sh b/client/rebuild-th-kernel.sh
new file mode 100644 (file)
index 0000000..4d06c5b
--- /dev/null
@@ -0,0 +1,101 @@
+#!/bin/sh
+
+dir=$(cd "$(dirname "$0")"; pwd)
+rpmdir=$(rpm -E %_topdir)
+dist=th
+
+pkgs_head="
+       dahdi-linux
+       ipset
+       iscsitarget
+       lirc
+       madwifi-ng
+       open-vm-tools
+       r8168
+       VirtualBox
+       xorg-driver-video-nvidia
+       xorg-driver-video-nvidia-legacy3
+       xtables-addons
+       xorg-driver-video-fglrx
+"
+
+pkgs_longterm="
+       $pkgs_head
+       openvswitch
+"
+pkgs_longterm_only="
+       e1000e
+       igb
+"
+
+# autotag from rpm-build-macros
+# displays latest used tag for a specfile
+autotag() {
+       local out s
+       for s in "$@"; do
+               # strip branches
+               s=${s%:*}
+               # ensure package ends with .spec
+               s=${s%.spec}.spec
+               out=$(cvs status -v $s | awk "!/Sticky/&&/auto-$dist-/{if (!a++) print \$1}")
+               echo "$s:$out"
+       done
+}
+
+get_last_tags() {
+       local pkg spec
+
+       echo >&2 "Fetching package tags: $*..."
+       for pkg in "$@"; do
+               echo >&2 "$pkg... "
+               if [ ! -e $pkg/$pkg.spec ]; then
+                       $rpmdir/builder -g $pkg -ns -r HEAD 1>&2
+               fi
+               if [ ! -e $pkg/$pkg.spec ]; then
+                       # just print it out, to fallback to base pkg name
+                       echo "$pkg"
+               else
+                       spec=$(autotag $pkg/$pkg.spec)
+                       echo >&2 "... $spec"
+                       echo $spec
+               fi
+       done
+}
+
+cd $rpmdir
+case "$1" in
+       head)
+               for pkg in $pkgs_head; do
+                       $rpmdir/builder -g $pkg -ns
+                       echo ./relup.sh -ui $a/$a.spec && make-request.sh -d th $a.spec
+               done
+               ;;
+       longterm)
+               cd $rpmdir
+               specs=$(get_last_tags $pkgs_longterm)
+               $dir/make-request.sh -r -d $dist --kernel longterm --without userspace $specs
+
+               specs=$pkgs_longterm_only
+               $dir/make-request.sh -r -d $dist --kernel longterm $specs
+               ;;
+       *)
+               # try to parse all args, filling them with last autotag
+               while [ $# -gt 0 ]; do
+                       case "$1" in
+                       --kernel|--with|--without)
+                               args="$1 $2"
+                               shift
+                               ;;
+                       -*)
+                               args="$args $1"
+                               ;;
+                       *)
+                               specs="$specs $1"
+                               ;;
+                       esac
+                       shift
+               done
+               specs=$(get_last_tags $specs)
+               $dir/make-request.sh -r -d $dist $args $specs
+               ;;
+esac
diff --git a/client/send-last-th-tag.sh b/client/send-last-th-tag.sh
new file mode 100644 (file)
index 0000000..1d56bd5
--- /dev/null
@@ -0,0 +1,35 @@
+#!/bin/sh
+arg="$@"
+
+specs=""
+opt=""
+
+for i in $arg; do
+       case "$i" in
+               *.spec)
+               specs="$specs $1"
+               shift
+               ;;
+               *)
+               opt="$opt $i"
+               shift
+               ;;
+       esac
+done
+
+for i in $specs; do
+       dir=$(dirname $i)
+       pkg=$(basename $i)
+       cd $dir || exit 1
+       specfile="$pkg"
+       tag=$(cvs status -v $specfile |grep "th-" | head -n 1 | awk ' { print $1 } ')
+       if [ -z "$tag" ]; then
+               echo "Th tag not found for $specfile."
+               continue
+       fi
+       echo "Rebuilding $i from tag $tag..."
+       set -x
+       make-request.sh -d th $opt $pkg:$tag
+       set +x
+done
+
index 38ab97f9f4bd7ea409bf8ee419b91de37252d40e..89fd9d2637d8953f44de628b1e1862ac9ebe992d 100644 (file)
@@ -1,8 +1,9 @@
 SHELL=/bin/sh
+PATH=/sbin:/bin:/usr/sbin:/usr/bin
 MAILTO=root
 
 #* * * * * builder exec nice -n 19 /usr/share/pld-builder/bin/request-fetcher.sh
 #* * * * * builder exec nice -n 19 /usr/share/pld-builder/bin/load-balancer.sh
 #* * * * * builder exec nice -n 19 /usr/share/pld-builder/bin/file-sender.sh
 
-#0 0 * * * chroot /srv/chroot nice -n 19 tmpwatch -m 240 /var/cache/pld-builder/ready
+#0 0 * * * root chroot /srv/chroot nice -n 19 tmpwatch -m 240 /var/cache/pld-builder/ready
This page took 0.222193 seconds and 4 git commands to generate.