]> git.pld-linux.org Git - projects/pld-builder.new.git/blob - client/make-request.sh
Rewrite autotag for git
[projects/pld-builder.new.git] / client / make-request.sh
1 #!/bin/sh
2
3 # prevent "*" from being expanded in builders var
4 set -f
5
6 builders=
7 with=
8 without=
9 flags=
10 command=
11 command_flags=
12 gpg_opts=
13 default_branch='HEAD'
14 dist=
15 url=
16 no_depend=no
17 verbose=no
18 autotag=no
19
20 if [ -x /usr/bin/python ]; then
21         send_mode="python"
22 else
23         echo "No python present, using mail mode"
24         send_mode="mail"
25 fi
26
27 if [ -n "$HOME_ETC" ]; then
28         USER_CFG=$HOME_ETC/.requestrc
29 else
30         USER_CFG=$HOME/.requestrc
31 fi
32
33 if [ ! -f "$USER_CFG" ]; then
34         echo "Creating config file $USER_CFG. You *must* edit it."
35         cat > $USER_CFG <<EOF
36 priority=2
37 requester=deviloper@pld-linux.org
38 default_key=deviloper@pld-linux.org
39 send_mode="$send_mode"
40 url="$url"
41 mailer="/usr/sbin/sendmail -t"
42 gpg_opts=""
43 dist=th
44 url="http://src.th.pld-linux.org:1234/"
45
46 # defaults:
47 f_upgrade=yes
48 EOF
49 exit
50 fi
51
52 if [ -f "$USER_CFG" ]; then
53         . $USER_CFG
54         # legacy fallback
55         if [ "${distro:+set}" = "set" ]; then
56                 dist=$distro
57         fi
58 fi
59
60 # internal options, not to be overriden
61 specs=
62 df_fetch=no
63 upgrade_macros=no
64
65 # Set colors
66 c_star=$(tput setaf 2)
67 c_red=$(tput setaf 1)
68 c_norm=$(tput op)
69 msg() {
70         echo >&2 "${c_star}*${c_norm} $*"
71 }
72 red() {
73         echo "${c_red}$*${c_norm}"
74 }
75
76 die() {
77         echo >&2 "$0: $*"
78         exit 1
79 }
80
81 send_request() {
82         # switch to mail mode, if no url set
83         [ -z "$url" ] && send_mode="mail"
84
85         case "$send_mode" in
86         "mail")
87                 msg "Sending using mail mode"
88                 cat - | $mailer
89                 ;;
90         *)
91                 msg "Sending using http mode to $url"
92                 cat - | python -c '
93 import sys, socket, urllib2
94
95 try:
96         data = sys.stdin.read()
97         url = sys.argv[1]
98         socket.setdefaulttimeout(10)
99         req = urllib2.Request(url, data)
100         f = urllib2.urlopen(req)
101         f.close()
102 except Exception, e:
103         print >> sys.stderr, "Problem while sending request via HTTP: %s: %s" % (url, e)
104         sys.exit(1)
105 print >> sys.stdout, "Request queued via HTTP."
106 ' "$url"
107                 ;;
108         esac
109 }
110
111 # simple df_fetcher, based on packages/fetchsrc_request
112 # TODO: tcp (smtp) mode
113 # TODO: adjust for ~/.requestrc config
114 df_fetch() {
115         local specs="$@"
116
117         # Sending by
118         local MAILER='/usr/sbin/sendmail'
119         # MAILER='/usr/bin/msmtp'
120         # Sending via
121         local VIA="SENDMAIL"
122         #VIA="localhost"
123         local VIA_ARGS=""
124         #VIA_ARGS="some additional flags"
125         # e.g. for msmtp:
126         # VIA_ARGS='-a gmail'
127         #
128         # DISTFILES EMAIL
129         local DMAIL="distfiles@pld-linux.org"
130
131         local HOST=$(hostname -f)
132         local LOGIN=${requester%@*}
133
134         for spec in $specs; do
135                 local SPEC=$(echo "$spec" | sed -e 's|:.*||')
136                 local BRANCH=$(echo "$spec" | sed -e 's|.*:||')
137                 echo >&2 "Distfiles Request: $SPEC:$BRANCH via $MAILER ${VIA_ARGS:+ ($VIA_ARGS)}"
138                 cat <<-EOF | "$MAILER" -t -i $VIA_ARGS
139                         To: $DMAIL
140                         From: $LOGIN <$LOGIN@$HOST>
141                         Subject: fetchsrc_request notify
142                         X-CVS-Module: SPECS
143                         X-distfiles-request: yes
144                         X-Login: $LOGIN
145                         X-Spec: $SPEC
146                         X-Branch: $BRANCH
147                         X-Flags: force-reply
148
149                         .
150                         EOF
151         done
152 }
153
154 # autotag from rpm-build-macros
155 # displays latest used tag for a specfile
156 autotag() {
157         local out s
158         for s in "$@"; do
159                 # strip branches
160                 s=${s%:*}
161                 # ensure package ends with .spec
162                 s=${s%.spec}.spec
163                 local gitdir=$(dirname $s)/.git
164                 out=$(git --git-dir="$gitdir" for-each-ref --count=1 --sort=-committerdate refs/tags/auto/$dist \
165                         --format='%(refname:short)')
166                 echo "$s:$out"
167         done
168 }
169
170 # get autotag for specs
171 # WARNING: This may checkout some files from CVS
172 get_autotag() {
173         local pkg spec rpmdir
174
175         rpmdir=$(rpm -E %_topdir)
176         cd $rpmdir
177         for pkg in "$@"; do
178                 # strip branches
179                 pkg=${pkg%:*}
180                 # strip .spec extension
181                 pkg=${pkg%.spec}
182                 # checkout only if missing
183                 if [ ! -e $pkg/$pkg.spec ]; then
184                         $rpmdir/builder -g $pkg -ns -r HEAD 1>&2
185                 fi
186                 if [ ! -e $pkg/$pkg.spec ]; then
187                         # just print it out, to fallback to base pkg name
188                         echo "$pkg"
189                 else
190                         autotag $pkg/$pkg.spec
191                 fi
192         done
193 }
194
195 usage() {
196         cat <<EOF
197 Usage: make-request.sh [OPTION] ... [SPECFILE] ....
198
199 Mandatory arguments to long options are mandatory for short options too.
200
201       --config-file /path/to/config/file
202             Source additional config file (after $USER_CFG), useful when
203             when sending build requests to Ac/Th from the same account
204       -a
205             Try to use latest auto-tag for the spec when building
206             WARNING: This will checkout new files to your packages dir
207       -b 'BUILDER BUILDER ...',  --builder='BUILDER BUILDER ...'
208             Sends request to given builders (in 'version-arch' format)
209       --with VALUE, --without VALUE
210             Build package with(out) a given bcond
211       --kernel VALUE
212             set alt_kernel to VALUE
213       --target VALUE
214             set --target to VALUE
215       -s BUILD_ID, --skip BUILD_ID[,BUILD_ID][,BUILD_ID]
216             mark build ids on src builder to be skipped
217       --branch VALUE
218             specify default branch for specs in request
219       -t, --test-build
220             Performs a 'test-build'. Package will be uploaded to hidden .test-builds/
221             ftp tree and won't be upgraded on builders.
222       -r, --ready-build
223             Preforms a 'ready' build. Package will be built and uploaded to test/ ftp tree
224             (and later moved by release manager staff to ready/ and main ftp tree)
225       -u, --upgrade
226             Forces package upgrade (for use with -c or -q, not -t)
227       -n, --no-upgrade
228             Disables package upgrade (for use with -r)
229       -ni, --no-install-br
230             Do not install missing BuildRequires (--nodeps)
231       -nd, --no-depend
232             Do not add dependency of build jobs, each job in batch runs itself
233       -j, --jobs
234             Number of parallel jobs for single build
235       -f, --flag
236       -d, --dist DISTRIBUTION_ID
237             Specify value for \$dist
238       -df,  --distfiles-fetch[-request] PACKAGE
239             Send distfiles request to fetch sources for PACKAGE
240       -cf, --command-flag
241             Not yet documented
242       -c, --command
243             Executes a given command on builders (prepended to build jobs if build jobs included)
244       -C, --post-command
245             Executes a given command on builders (appended to build jobs if build jobs included)
246       --test-remove-pkg
247             shortcut for --command poldek -evt ARGS
248       --remove-pkg
249             shortcut for --command poldek -ev --noask ARGS
250       --upgrade-pkg
251             shortcut for --command poldek --up -Uv ARGS
252       --pull
253             Updates builders infrastructure (outside chroot)
254       --update-macros
255             Updates rpm-build-macros on src builder
256       -q
257             shortcut for --command rpm -q ARGS
258       -g, --gpg-opts "opts"
259             Pass additional options to gpg binary
260       -p, --priority VALUE
261             sets request priority (default 2)
262       -h, --help
263             Displays this help message
264 EOF
265         exit 0
266 }
267
268 # validate distro, set $dist
269 set_dist() {
270         case "$1" in
271         ac)
272                 ;;
273         ac-java|ac-xen)
274                 ;;
275         ti)
276                 ;;
277         ti-dev)
278                 ;;
279         th)
280                 ;;
281         th-java)
282                 ;;
283         aidath)
284                 ;;
285         *)
286                 die "dist \`$1' not known"
287                 ;;
288         esac
289
290         dist=$1
291 }
292
293 while [ $# -gt 0 ] ; do
294         case "$1" in
295                 -d | --dist | --distro)
296                         set_dist $2
297                         shift
298                         ;;
299
300                 --config-file)
301                         [ -f "$2" ] && . $2 || die "Config file not found"
302                         shift
303                         ;;
304
305                 --builder | -b)
306                         for b in $2; do
307                                 builders="$builders ${b%:*}"
308                         done
309                         shift
310                         ;;
311
312                 -a)
313                         autotag=yes
314                         ;;
315
316                 --with)
317                         with="$with $(echo "$2" | tr ',' ' ')"
318                         shift
319                         ;;
320
321                 --without)
322                         without="$without $(echo "$2" | tr ',' ' ')"
323                         shift
324                         ;;
325
326                 --test-build | -t)
327                         build_mode=test
328                         f_upgrade=no
329                         ;;
330
331                 --kernel)
332                         kernel=$2
333                         shift
334                         ;;
335
336                 --target)
337                         target=$2
338                         shift
339                         ;;
340
341                 -s|--skip)
342                         skip="$2"
343                         shift
344                         ;;
345
346                 --branch)
347                         branch=$2
348                         shift
349                         ;;
350
351                 --priority | -p)
352                         priority=$2
353                         shift
354                         ;;
355
356                 --ready-build | -r)
357                         build_mode=ready
358                         ;;
359
360                 --upgrade | -u)
361                         f_upgrade=yes
362                         ;;
363
364                 --no-upgrade | -n)
365                         f_upgrade=no
366                         ;;
367
368                 --no-depend | -nd)
369                         no_depend=yes
370                         ;;
371
372                 --no-install-br | -ni)
373                         flags="$flags no-install-br"
374                         ;;
375
376                 -j | --jobs)
377                         jobs="$2"
378                         shift
379                         ;;
380
381                 -j*)
382                         jobs="${1#-j}"
383                         ;;
384
385                 -v)
386                         verbose=yes
387                         ;;
388
389                 --flag | -f)
390                         flags="$flags $2"
391                         shift
392                         ;;
393
394                 --command-flags | -cf)
395                         command_flags="$2"
396                         shift
397                         ;;
398
399                 --command | -c)
400                         command="$2"
401                         if [ "$command" = - ]; then
402                                 echo >&2 "Reading command from STDIN"
403                                 echo >&2 "---"
404                                 command=$(cat)
405                                 echo >&2 "---"
406                         fi
407                         shift
408                         ;;
409                 --post-command | -C)
410                         post_command="$2"
411                         if [ "$post_command" = - ]; then
412                                 echo >&2 "Reading post_command from STDIN"
413                                 echo >&2 "---"
414                                 post_command=$(cat)
415                                 echo >&2 "---"
416                         fi
417                         shift
418                         ;;
419                 --test-remove-pkg)
420                         command="poldek -evt $2"
421                         f_upgrade=no
422                         shift
423                         ;;
424                 --remove-pkg)
425                         command="for a in $2; do poldek -ev --noask \$a; done"
426                         f_upgrade=no
427                         shift
428                         ;;
429                 --upgrade-pkg|-Uhv)
430                         command="poldek --up; poldek -uv $2"
431                         f_upgrade=no
432                         shift
433                         ;;
434                 -q)
435                         command="rpm -q $2"
436                         f_upgrade=no
437                         shift
438                         ;;
439
440                 --pull)
441                         command_flags="no-chroot"
442                         command="git pull"
443                         f_upgrade=no
444                         ;;
445
446                 --update-macros)
447                         upgrade_macros="yes"
448                         ;;
449
450                 -df | --distfiles-fetch | --distfiles-fetch-request)
451                         df_fetch=yes
452                         ;;
453
454                 --gpg-opts | -g)
455                         gpg_opts="$2"
456                         shift
457                         ;;
458
459                 --help | -h)
460                         usage
461                         ;;
462
463                 -*)
464                         die "unknown knob: $1"
465                         ;;
466
467                 *:* | *)
468                         specs="$specs $1"
469                         ;;
470         esac
471         shift
472 done
473
474 case "$dist" in
475 ac)
476         builder_email="builder-ac@pld-linux.org"
477         default_builders="ac-*"
478         default_branch="AC-branch"
479         url="http://ep09.pld-linux.org:1289/"
480         control_url="http://ep09.pld-linux.org/~buildsrc"
481         ;;
482 ac-java) # fake "dist" for java available ac architectures
483         builder_email="builder-ac@pld-linux.org"
484         default_builders="ac-i586 ac-i686 ac-athlon ac-amd64"
485         default_branch="AC-branch"
486         url="http://ep09.pld-linux.org:1289/"
487         ;;
488 ac-xen) # fake "dist" for xen-enabled architectures
489         builder_email="builder-ac@pld-linux.org"
490         default_builders="ac-i686 ac-athlon ac-amd64"
491         default_branch="AC-branch"
492         ;;
493 ti)
494         builder_email="builderti@ep09.pld-linux.org"
495         default_builders="ti-*"
496         url="http://ep09.pld-linux.org:1231/"
497         control_url="http://ep09.pld-linux.org/~builderti"
498         ;;
499 ti-dev)
500         builder_email="buildertidev@ep09.pld-linux.org"
501         default_builders="ti-dev-*"
502         url="http://ep09.pld-linux.org:1232/"
503         control_url="http://ep09.pld-linux.org/~buildertidev"
504         ;;
505 th)
506         builder_email="builderth@pld-linux.org"
507         default_builders="th-*"
508         url="http://src.th.pld-linux.org:1234/"
509         control_url="http://src.th.pld-linux.org"
510         ;;
511 th-java) # fake "dist" for java available th architectures
512         builder_email="builderth@pld-linux.org"
513         default_builders="th-x86_64 th-athlon th-i686"
514         url="http://src.th.pld-linux.org:1234/"
515         ;;
516 aidath)
517         builder_email="builderaidath@ep09.pld-linux.org"
518         default_builders="aidath-*"
519         ;;
520 *)
521         die "dist \`$dist' not known"
522         ;;
523 esac
524
525 # need to do this after dist selection
526 if [ "$skip" ]; then
527         skip=$(skip="$skip" control_url="$control_url" python -c '
528 import urllib2
529 import sys
530 import StringIO
531 import gzip
532 import re
533 import os
534 import string
535 from xml.dom import minidom
536
537 skip = os.environ.get("skip").split(",");
538 control_url = os.environ.get("control_url")
539
540 print >> sys.stderr, "* Check queue_id-s against %s" % control_url
541
542 try:
543         headers = { "Cache-Control": "no-cache", "Pragma": "no-cache" }
544         req = urllib2.Request(url=control_url + "/queue.gz", headers=headers)
545         f = urllib2.urlopen(req)
546 except Exception, e:
547         print >> sys.stderr, "Fetch error %s: %s" % (control_url + "/queue.gz", e)
548         sys.exit(1)
549
550 sio = StringIO.StringIO()
551 sio.write(f.read())
552 f.close()
553 sio.seek(0)
554 f = gzip.GzipFile(fileobj = sio)
555
556 xml = re.compile("(<queue>.*?</queue>)", re.DOTALL).match(f.read()).group(1)
557 d = minidom.parseString(xml)
558
559 q = []
560 for c in d.documentElement.childNodes:
561         if c.nodeName != "group":
562                 continue
563         q.append(c.attributes["id"].value)
564
565 err = 0
566 for s in skip:
567         if s not in q:
568                 print >> sys.stderr, "- Check %s: ERROR: Not valid queue-id" % s
569                 err = 1
570         else:
571                 print >> sys.stderr, "- Check %s: OK" % s
572 if err == 1:
573         sys.exit(1)
574 print string.join(skip, ",")
575 ') || exit $?
576         f_upgrade=no
577         build_mode=test
578         priority=-1
579         command="skip:$skip"
580         command_flags="no-chroot"
581         builders="$dist-src"
582 fi
583
584 branch=${branch:-$default_branch}
585
586 specs=`for s in $specs; do
587         case "$s" in
588         ^)
589                 # skip marker - pass it along
590                 echo $s
591                 ;;
592         *.spec:*) # spec with branch
593                 basename $s
594                 ;;
595         *.spec) # spec without branch
596                 echo $(basename $s):$branch
597                 ;;
598         *:*) # package name with branch
599                 basename $s | sed -e 's/:/.spec:/'
600                 ;;
601         *) # just package name
602                 echo $(basename $s).spec:$branch
603                 ;;
604         esac
605 done`
606
607 if [ "$autotag" = "yes" ]; then
608         msg "Auto autotag build enabled"
609         specs=$(get_autotag $specs)
610 fi
611
612 if [ "$df_fetch" = "yes" ]; then
613         df_fetch $specs
614         exit 0
615 fi
616
617 if [ "$upgrade_macros" = "yes" ]; then
618         command="poldek --up; poldek -uv rpm-build-macros"
619         builders="$dist-src"
620         f_upgrade=no
621         build_mode=test
622 fi
623
624 if [[ "$requester" != *@* ]] ; then
625         requester="$requester@pld-linux.org"
626 fi
627
628 if [ -z "$builders" ] ; then
629         builders="$default_builders"
630 fi
631
632 if [ "$f_upgrade" = "yes" ] ; then
633         flags="$flags upgrade"
634 fi
635
636 if [ "$build_mode" = "test" ] ; then
637         if [ "$f_upgrade" = "yes" ] ; then
638                 die "--upgrade and --test-build are mutually exclusive"
639         fi
640         flags="$flags test-build"
641 fi
642
643 if [ -z "$build_mode" ] ; then
644         # missing build mode, builders go crazy when you proceed"
645         die "please specify build mode"
646 fi
647
648
649 ok=
650 for s in $specs; do
651         ok=1
652 done
653
654 if [ -z "$specs" -a -z "$command" ]; then
655         die "no packages to build or command to invoke specified"
656 fi
657
658 id=$(uuidgen)
659
660 gen_req() {
661         echo "<group id='$id' no='0' flags='$flags'>"
662         echo "  <time>$(date +%s)</time>"
663         msg "Using priority $priority"
664         echo "  <priority>$priority</priority>"
665         if [ -n "$jobs" ]; then
666                 msg "Using jobs $jobs"
667                 echo "  <maxjobs>$jobs</maxjobs>"
668         fi
669         if [ -z "$url" ]; then
670                 msg "Using email $builder_email"
671         else
672                 msg "Using URL $url"
673         fi
674
675         if [ "$build_mode" = "ready" ]; then
676                 msg "Build mode: $(tput setaf 2)$build_mode$c_norm"
677         else
678                 msg "Build mode: $(tput setaf 3)$build_mode$c_norm"
679         fi
680
681         msg "Queue-ID: $id"
682         echo
683
684         # job to depend on
685         local depend=
686         local b i=1
687         local name branch builders_xml
688
689         for b in $builders; do
690                 msg "Builder: $(red $b)"
691                 builders_xml="$builders_xml <builder>$b</builder>"
692         done
693
694         if [ "$command" ]; then
695                 bid=$(uuidgen)
696                 echo -E >&2 "* Command: $command"
697                 echo "  <batch id='$bid' depends-on=''>"
698                 echo "           <command flags='$command_flags'>"
699                 echo -E "$command" | sed -e 's,&,\&amp;,g;s,<,\&lt;,g;s,>,\&gt;,g'
700                 echo "</command>"
701                 echo "           <info></info>"
702                 echo "$builders_xml"
703                 echo "  </batch>"
704                 depend=$bid
705         fi
706
707         if [ "$f_upgrade" = "yes" ] ; then
708                 msg "Upgrade mode: $f_upgrade"
709         fi
710
711         for s in $specs; do
712                 # skip marker
713                 if [ "$s" = "^" ]; then
714                         depend=
715                         continue
716                 fi
717                 if [ "$no_depend" = yes ]; then
718                         depend=
719                 fi
720                 bid=$(uuidgen)
721                 echo "  <batch id='$bid' depends-on='$depend'>"
722
723                 name=$(echo "$s" | sed -e 's|:.*||')
724                 branch=$(echo "$s" | sed -e 's|.*:||')
725                 msg "Adding #$i $name:$branch${kernel:+ alt_kernel=$kernel}${target:+ target=$target}${depend:+ depends on $depend}"
726                 echo "           <spec>$name</spec>"
727                 echo "           <branch>$branch</branch>"
728                 echo "           ${kernel:+<kernel>$kernel</kernel>}"
729                 echo "           ${target:+<target>$target</target>}"
730                 echo "           <info></info>"
731                 echo
732                 for b in $with; do
733                         echo "           <with>$b</with>"
734                 done
735                 for b in $without; do
736                         echo "           <without>$b</without>"
737                 done
738                 echo
739                 echo "$builders_xml"
740                 echo "  </batch>"
741                 i=$((i+1))
742
743                 # let next job depend on previous
744                 depend=$bid
745         done
746
747         if [ "$post_command" ]; then
748                 bid=$(uuidgen)
749                 if [ "$no_depend" = yes ]; then
750                         depend=
751                 fi
752                 echo -E >&2 "* Post-Command: $post_command"
753                 echo "  <batch id='$bid' depends-on='$depend'>"
754                 echo "           <command flags='$command_flags'>"
755                 echo -E "$post_command" | sed -e 's,&,\&amp;,g;s,<,\&lt;,g;s,>,\&gt;,g'
756                 echo "</command>"
757                 echo "           <info></info>"
758                 echo "$builders_xml"
759                 echo "  </batch>"
760                 depend=$bid
761         fi
762
763         echo "</group>"
764 }
765
766 gen_email () {
767         # make request first, so the STDERR/STDOUT streams won't be mixed
768         local tmp req
769         tmp=$(mktemp)
770         gen_req > $tmp
771
772         if [ "$verbose" = "yes" ]; then
773                 cat $tmp >&2
774         fi
775
776         cat <<-EOF
777         From: $requester
778         To: $builder_email
779         Subject: build request
780         Message-Id: <$id@$(hostname)>
781         X-New-PLD-Builder: request
782         X-Requester-Version: \$Id$
783
784         EOF
785
786         gpg --clearsign --default-key $default_key $gpg_opts --output=- $tmp
787         rm -f $tmp
788 }
789
790 gen_email | send_request
This page took 0.197323 seconds and 3 git commands to generate.