client/make-request.sh

   1 #!/bin/sh
   2
   3 # prevent "*" from being expanded in builders var
   4 set -f
   5
   6 builders=
   7 with=
   8 without=
   9 flags=
  10 command=
  11 command_flags=
  12 gpg_opts=
  13 default_branch='HEAD'
  14 distro=
  15 url=
  16 no_depend=no
  17 verbose=no
  18
  19 if [ -x /usr/bin/python ]; then
  20         send_mode="python"
  21 else
  22         echo "No python present, using mail mode"
  23         send_mode="mail"
  24 fi
  25
  26 if [ -n "$HOME_ETC" ]; then
  27         USER_CFG=$HOME_ETC/.requestrc
  28 else
  29         USER_CFG=$HOME/.requestrc
  30 fi
  31
  32 if [ ! -f "$USER_CFG" ]; then
  33         echo "Creating config file $USER_CFG. You *must* edit it."
  34         cat > $USER_CFG <<EOF
  35 priority=2
  36 requester=deviloper@pld-linux.org
  37 default_key=deviloper@pld-linux.org
  38 send_mode="$send_mode"
  39 url="$url"
  40 mailer="/usr/sbin/sendmail -t"
  41 gpg_opts=""
  42 distro=th
  43 url="http://src.th.pld-linux.org:1234/"
  44
  45 # defaults:
  46 f_upgrade=yes
  47 EOF
  48 exit
  49 fi
  50
  51 if [ -f "$USER_CFG" ]; then
  52         . $USER_CFG
  53 fi
  54
  55 # internal options, not to be overriden
  56 specs=
  57 df_fetch=no
  58 upgrade_macros=no
  59
  60 # Set colors
  61 c_star=$(tput setaf 2)
  62 c_red=$(tput setaf 1)
  63 c_norm=$(tput op)
  64 msg() {
  65         echo >&2 "${c_star}*${c_norm} $*"
  66 }
  67 red() {
  68         echo "${c_red}$*${c_norm}"
  69 }
  70
  71 die() {
  72         echo >&2 "$0: $*"
  73         exit 1
  74 }
  75
  76 send_request() {
  77         # switch to mail mode, if no url set
  78         [ -z "$url" ] && send_mode="mail"
  79
  80         case "$send_mode" in
  81         "mail")
  82                 msg "Sending using mail mode"
  83                 cat - | $mailer
  84                 ;;
  85         *)
  86                 msg "Sending using http mode to $url"
  87                 cat - | python -c '
  88 import sys, socket, urllib2
  89
  90 try:
  91         data = sys.stdin.read()
  92         url = sys.argv[1]
  93         socket.setdefaulttimeout(10)
  94         req = urllib2.Request(url, data)
  95         f = urllib2.urlopen(req)
  96         f.close()
  97 except Exception, e:
  98         print >> sys.stderr, "Problem while sending request via HTTP: %s: %s" % (url, e)
  99         sys.exit(1)
 100 print >> sys.stdout, "Request queued via HTTP."
 101 ' "$url"
 102                 ;;
 103         esac
 104 }
 105
 106 # simple df_fetcher, based on packages/fetchsrc_request
 107 # TODO: tcp (smtp) mode
 108 # TODO: adjust for ~/.requestrc config
 109 df_fetch() {
 110         local specs="$@"
 111
 112         # Sending by
 113         local MAILER='/usr/sbin/sendmail'
 114         # MAILER='/usr/bin/msmtp'
 115         # Sending via
 116         local VIA="SENDMAIL"
 117         #VIA="localhost"
 118         local VIA_ARGS=""
 119         #VIA_ARGS="some additional flags"
 120         # e.g. for msmtp:
 121         # VIA_ARGS='-a gmail'
 122         #
 123         # DISTFILES EMAIL
 124         local DMAIL="distfiles@pld-linux.org"
 125
 126         local HOST=$(hostname -f)
 127         local LOGIN=${requester%@*}
 128
 129         for spec in $specs; do
 130                 local SPEC=$(echo "$spec" | sed -e 's|:.*||')
 131                 local BRANCH=$(echo "$spec" | sed -e 's|.*:||')
 132                 echo >&2 "Distfiles Request: $SPEC:$BRANCH via $MAILER ${VIA_ARGS:+ ($VIA_ARGS)}"
 133                 cat <<-EOF | "$MAILER" -t -i $VIA_ARGS
 134                         To: $DMAIL
 135                         From: $LOGIN <$LOGIN@$HOST>
 136                         Subject: fetchsrc_request notify
 137                         X-CVS-Module: SPECS
 138                         X-distfiles-request: yes
 139                         X-Login: $LOGIN
 140                         X-Spec: $SPEC
 141                         X-Branch: $BRANCH
 142                         X-Flags: force-reply
 143
 144                         .
 145                         EOF
 146         done
 147 }
 148
 149 usage() {
 150         cat <<EOF
 151 Usage: make-request.sh [OPTION] ... [SPECFILE] ....
 152
 153 Mandatory arguments to long options are mandatory for short options too.
 154
 155       --config-file /path/to/config/file
 156             Source additional config file (after $USER_CFG), useful when
 157             when sending build requests to Ac/Th from the same account
 158       -b 'BUILDER BUILDER ...',  --builder='BUILDER BUILDER ...'
 159            Sends request to given builders (in 'version-arch' format)
 160       --with VALUE, --without VALUE
 161             Build package with(out) a given bcond
 162       --kernel VALUE
 163             set alt_kernel to VALUE
 164       --target VALUE
 165             set --target to VALUE
 166       -s BUILD_ID, --skip BUILD_ID[,BUILD_ID][,BUILD_ID]
 167             mark build ids on src builder to be skipped
 168       --branch VALUE
 169             specify default branch for specs in request
 170       -t, --test-build
 171             Performs a 'test-build'. Package will be uploaded to hidden .test-builds/
 172             ftp tree and won't be upgraded on builders.
 173       -r, --ready-build
 174             Preforms a 'ready' build. Package will be built and uploaded to test/ ftp tree
 175             (and later moved by release manager staff to ready/ and main ftp tree)
 176       -u, --upgrade
 177             Forces package upgrade (for use with -c or -q, not -t)
 178       -n, --no-upgrade
 179             Disables package upgrade (for use with -r)
 180       -ni, --no-install-br
 181             Do not install missing BuildRequires (--nodeps)
 182       -nd, --no-depend
 183             Do not add dependency of build jobs, each job in batch runs itself
 184       -j, --jobs
 185             Number of parallel jobs for single build
 186       -f, --flag
 187       -d, --distro DISTRO
 188             Specify value for \$distro
 189       -df,  --distfiles-fetch[-request] PACKAGE
 190             Send distfiles request to fetch sources for PACKAGE
 191       -cf, --command-flag
 192             Not yet documented
 193       -c, --command
 194             Executes a given command on builders (prepended to build jobs if build jobs included)
 195       -C, --post-command
 196             Executes a given command on builders (appended to build jobs if build jobs included)
 197       --test-remove-pkg
 198             shortcut for --command poldek -evt ARGS
 199       --remove-pkg
 200             shortcut for --command poldek -ev --noask ARGS
 201       --upgrade-pkg
 202             shortcut for --command poldek --up -Uv ARGS
 203       --pull
 204             Updates builders infrastructure (outside chroot)
 205       --update-macros
 206             Updates rpm-build-macros on src builder
 207       -q
 208             shortcut for --command rpm -q ARGS
 209       -g, --gpg-opts "opts"
 210             Pass additional options to gpg binary
 211       -p, --priority VALUE
 212             sets request priority (default 2)
 213       -h, --help
 214             Displays this help message
 215 EOF
 216         exit 0
 217 }
 218
 219 # validate distro, set $distro
 220 set_distro() {
 221         case "$1" in
 222         ac)
 223                 ;;
 224         ac-java|ac-xen)
 225                 ;;
 226         ti)
 227                 ;;
 228         ti-dev)
 229                 ;;
 230         th)
 231                 ;;
 232         th-java)
 233                 ;;
 234         aidath)
 235                 ;;
 236         *)
 237                 die "distro \`$1' not known"
 238                 ;;
 239         esac
 240
 241         distro=$1
 242 }
 243
 244 while [ $# -gt 0 ] ; do
 245         case "$1" in
 246                 --distro | -d)
 247                         set_distro $2
 248                         shift
 249                         ;;
 250
 251                 --config-file)
 252                         [ -f "$2" ] && . $2 || die "Config file not found"
 253                         shift
 254                         ;;
 255
 256                 --builder | -b)
 257                         for b in $2; do
 258                                 builders="$builders ${b%:*}"
 259                         done
 260                         shift
 261                         ;;
 262
 263                 --with)
 264                         with="$with $(echo "$2" | tr ',' ' ')"
 265                         shift
 266                         ;;
 267
 268                 --without)
 269                         without="$without $(echo "$2" | tr ',' ' ')"
 270                         shift
 271                         ;;
 272
 273                 --test-build | -t)
 274                         build_mode=test
 275                         f_upgrade=no
 276                         ;;
 277
 278                 --kernel)
 279                         kernel=$2
 280                         shift
 281                         ;;
 282
 283                 --target)
 284                         target=$2
 285                         shift
 286                         ;;
 287
 288                 -s|--skip)
 289                         skip="$2"
 290                         shift
 291                         ;;
 292
 293                 --branch)
 294                         branch=$2
 295                         shift
 296                         ;;
 297
 298                 --priority | -p)
 299                         priority=$2
 300                         shift
 301                         ;;
 302
 303                 --ready-build | -r)
 304                         build_mode=ready
 305                         ;;
 306
 307                 --upgrade | -u)
 308                         f_upgrade=yes
 309                         ;;
 310
 311                 --no-upgrade | -n)
 312                         f_upgrade=no
 313                         ;;
 314
 315                 --no-depend | -nd)
 316                         no_depend=yes
 317                         ;;
 318
 319                 --no-install-br | -ni)
 320                         flags="$flags no-install-br"
 321                         ;;
 322
 323                 -j | --jobs)
 324                         jobs="$2"
 325                         shift
 326                         ;;
 327
 328                 -j*)
 329                         jobs="${1#-j}"
 330                         ;;
 331
 332                 -v)
 333                         verbose=yes
 334                         ;;
 335
 336                 --flag | -f)
 337                         flags="$flags $2"
 338                         shift
 339                         ;;
 340
 341                 --command-flags | -cf)
 342                         command_flags="$2"
 343                         shift
 344                         ;;
 345
 346                 --command | -c)
 347                         command="$2"
 348                         if [ "$command" = - ]; then
 349                                 echo >&2 "Reading command from STDIN"
 350                                 echo >&2 "---"
 351                                 command=$(cat)
 352                                 echo >&2 "---"
 353                         fi
 354                         shift
 355                         ;;
 356                 --post-command | -C)
 357                         post_command="$2"
 358                         if [ "$post_command" = - ]; then
 359                                 echo >&2 "Reading post_command from STDIN"
 360                                 echo >&2 "---"
 361                                 post_command=$(cat)
 362                                 echo >&2 "---"
 363                         fi
 364                         shift
 365                         ;;
 366                 --test-remove-pkg)
 367                         command="poldek -evt $2"
 368                         f_upgrade=no
 369                         shift
 370                         ;;
 371                 --remove-pkg)
 372                         command="for a in $2; do poldek -ev --noask \$a; done"
 373                         f_upgrade=no
 374                         shift
 375                         ;;
 376                 --upgrade-pkg|-Uhv)
 377                         command="poldek --up; poldek -uv $2"
 378                         f_upgrade=no
 379                         shift
 380                         ;;
 381                 -q)
 382                         command="rpm -q $2"
 383                         f_upgrade=no
 384                         shift
 385                         ;;
 386
 387                 --pull)
 388                         command_flags="no-chroot"
 389                         command="git pull"
 390                         f_upgrade=no
 391                         ;;
 392
 393                 --update-macros)
 394                         upgrade_macros="yes"
 395                         ;;
 396
 397                 -df | --distfiles-fetch | --distfiles-fetch-request)
 398                         df_fetch=yes
 399                         ;;
 400
 401                 --gpg-opts | -g)
 402                         gpg_opts="$2"
 403                         shift
 404                         ;;
 405
 406                 --help | -h)
 407                         usage
 408                         ;;
 409
 410                 -*)
 411                         die "unknown knob: $1"
 412                         ;;
 413
 414                 *:* | *)
 415                         specs="$specs $1"
 416                         ;;
 417         esac
 418         shift
 419 done
 420
 421 case "$distro" in
 422 ac)
 423         builder_email="builder-ac@pld-linux.org"
 424         default_builders="ac-*"
 425         default_branch="AC-branch"
 426         url="http://ep09.pld-linux.org:1289/"
 427         control_url="http://ep09.pld-linux.org/~buildsrc"
 428         ;;
 429 ac-java) # fake "distro" for java available ac architectures
 430         builder_email="builder-ac@pld-linux.org"
 431         default_builders="ac-i586 ac-i686 ac-athlon ac-amd64"
 432         default_branch="AC-branch"
 433         url="http://ep09.pld-linux.org:1289/"
 434         ;;
 435 ac-xen) # fake "distro" for xen-enabled architectures
 436         builder_email="builder-ac@pld-linux.org"
 437         default_builders="ac-i686 ac-athlon ac-amd64"
 438         default_branch="AC-branch"
 439         ;;
 440 ti)
 441         builder_email="builderti@ep09.pld-linux.org"
 442         default_builders="ti-*"
 443         url="http://ep09.pld-linux.org:1231/"
 444         control_url="http://ep09.pld-linux.org/~builderti"
 445         ;;
 446 ti-dev)
 447         builder_email="buildertidev@ep09.pld-linux.org"
 448         default_builders="ti-dev-*"
 449         url="http://ep09.pld-linux.org:1232/"
 450         control_url="http://ep09.pld-linux.org/~buildertidev"
 451         ;;
 452 th)
 453         builder_email="builderth@pld-linux.org"
 454         default_builders="th-*"
 455         url="http://src.th.pld-linux.org:1234/"
 456         control_url="http://src.th.pld-linux.org"
 457         ;;
 458 th-java) # fake "distro" for java available th architectures
 459         builder_email="builderth@pld-linux.org"
 460         default_builders="th-x86_64 th-athlon th-i686"
 461         url="http://src.th.pld-linux.org:1234/"
 462         ;;
 463 aidath)
 464         builder_email="builderaidath@ep09.pld-linux.org"
 465         default_builders="aidath-*"
 466         ;;
 467 *)
 468         die "distro \`$distro' not known"
 469         ;;
 470 esac
 471
 472 # need to do this after distro selection
 473 if [ "$skip" ]; then
 474         skip=$(skip="$skip" control_url="$control_url" python -c '
 475 import urllib2
 476 import sys
 477 import StringIO
 478 import gzip
 479 import re
 480 import os
 481 import string
 482 from xml.dom import minidom
 483
 484 skip = os.environ.get("skip").split(",");
 485 control_url = os.environ.get("control_url")
 486
 487 print >> sys.stderr, "* Check queue_id-s against %s" % control_url
 488
 489 try:
 490         headers = { "Cache-Control": "no-cache", "Pragma": "no-cache" }
 491         req = urllib2.Request(url=control_url + "/queue.gz", headers=headers)
 492         f = urllib2.urlopen(req)
 493 except Exception, e:
 494         print >> sys.stderr, "Fetch error %s: %s" % (control_url + "/queue.gz", e)
 495         sys.exit(1)
 496
 497 sio = StringIO.StringIO()
 498 sio.write(f.read())
 499 f.close()
 500 sio.seek(0)
 501 f = gzip.GzipFile(fileobj = sio)
 502
 503 xml = re.compile("(<queue>.*?</queue>)", re.DOTALL).match(f.read()).group(1)
 504 d = minidom.parseString(xml)
 505
 506 q = []
 507 for c in d.documentElement.childNodes:
 508         if c.nodeName != "group":
 509                 continue
 510         q.append(c.attributes["id"].value)
 511
 512 err = 0
 513 for s in skip:
 514         if s not in q:
 515                 print >> sys.stderr, "- Check %s: ERROR: Not valid queue-id" % s
 516                 err = 1
 517         else:
 518                 print >> sys.stderr, "- Check %s: OK" % s
 519 if err == 1:
 520         sys.exit(1)
 521 print string.join(skip, ",")
 522 ') || exit $?
 523         f_upgrade=no
 524         build_mode=test
 525         priority=-1
 526         command="skip:$skip"
 527         command_flags="no-chroot"
 528         builders="$distro-src"
 529 fi
 530
 531 branch=${branch:-$default_branch}
 532
 533 specs=`for s in $specs; do
 534         case "$s" in
 535         ^)
 536                 # skip marker
 537                 echo $s
 538                 ;;
 539         *.spec:*) # spec with branch
 540                 basename $s
 541                 ;;
 542         *.spec) # spec without branch
 543                 echo $(basename $s):$branch
 544                 ;;
 545         *:*) # package name with branch
 546                 basename $s | sed -e 's/:/.spec:/'
 547                 ;;
 548         *) # just package name
 549                 echo $(basename $s).spec:$branch
 550                 ;;
 551         esac
 552 done`
 553
 554 if [ "$df_fetch" = "yes" ]; then
 555         df_fetch $specs
 556         exit 0
 557 fi
 558
 559 if [ "$upgrade_macros" = "yes" ]; then
 560         command="poldek --up; poldek -uv rpm-build-macros"
 561         builders="$distro-src"
 562         f_upgrade=no
 563         build_mode=test
 564 fi
 565
 566 if [[ "$requester" != *@* ]] ; then
 567         requester="$requester@pld-linux.org"
 568 fi
 569
 570 if [ -z "$builders" ] ; then
 571         builders="$default_builders"
 572 fi
 573
 574 if [ "$f_upgrade" = "yes" ] ; then
 575         flags="$flags upgrade"
 576 fi
 577
 578 if [ "$build_mode" = "test" ] ; then
 579         if [ "$f_upgrade" = "yes" ] ; then
 580                 die "--upgrade and --test-build are mutually exclusive"
 581         fi
 582         flags="$flags test-build"
 583 fi
 584
 585 if [ -z "$build_mode" ] ; then
 586         # missing build mode, builders go crazy when you proceed"
 587         die "please specify build mode"
 588 fi
 589
 590
 591 ok=
 592 for s in $specs; do
 593         ok=1
 594 done
 595
 596 if [ -z "$specs" -a -z "$command" ]; then
 597         die "no packages to build or command to invoke specified"
 598 fi
 599
 600 id=$(uuidgen)
 601
 602 gen_req() {
 603         echo "<group id='$id' no='0' flags='$flags'>"
 604         echo "  <time>$(date +%s)</time>"
 605         msg "Using priority $priority"
 606         echo "  <priority>$priority</priority>"
 607         if [ -n "$jobs" ]; then
 608                 msg "Using jobs $jobs"
 609                 echo "  <maxjobs>$jobs</maxjobs>"
 610         fi
 611         msg "Build mode: $build_mode"
 612         if [ -z "$url" ]; then
 613                 msg "Using email $builder_email"
 614         else
 615                 msg "Using URL $url"
 616         fi
 617         msg "Queue-ID: $id"
 618         echo
 619
 620         # job to depend on
 621         local depend=
 622         local b i=1
 623         local name branch builders_xml
 624
 625         for b in $builders; do
 626                 msg "Builder: $(red $b)"
 627                 builders_xml="$builders_xml <builder>$b</builder>"
 628         done
 629
 630         if [ "$command" ]; then
 631                 bid=$(uuidgen)
 632                 echo -E >&2 "* Command: $command"
 633                 echo "  <batch id='$bid' depends-on=''>"
 634                 echo "           <command flags='$command_flags'>"
 635                 echo -E "$command" | sed -e 's,&,\&amp;,g;s,<,\&lt;,g;s,>,\&gt;,g'
 636                 echo "</command>"
 637                 echo "           <info></info>"
 638                 echo "$builders_xml"
 639                 echo "  </batch>"
 640                 depend=$bid
 641         fi
 642
 643         if [ "$f_upgrade" = "yes" ] ; then
 644                 msg "Upgrade mode: $f_upgrade"
 645         fi
 646
 647         for s in $specs; do
 648                 # skip marker
 649                 if [ "$s" = "^" ]; then
 650                         depend=
 651                         continue
 652                 fi
 653                 if [ "$no_depend" = yes ]; then
 654                         depend=
 655                 fi
 656                 bid=$(uuidgen)
 657                 echo "  <batch id='$bid' depends-on='$depend'>"
 658
 659                 name=$(echo "$s" | sed -e 's|:.*||')
 660                 branch=$(echo "$s" | sed -e 's|.*:||')
 661                 msg "Adding #$i $name:$branch${kernel:+ alt_kernel=$kernel}${target:+ target=$target}${depend:+ depends on $depend}"
 662                 echo "           <spec>$name</spec>"
 663                 echo "           <branch>$branch</branch>"
 664                 echo "           ${kernel:+<kernel>$kernel</kernel>}"
 665                 echo "           ${target:+<target>$target</target>}"
 666                 echo "           <info></info>"
 667                 echo
 668                 for b in $with; do
 669                         echo "           <with>$b</with>"
 670                 done
 671                 for b in $without; do
 672                         echo "           <without>$b</without>"
 673                 done
 674                 echo
 675                 echo "$builders_xml"
 676                 echo "  </batch>"
 677                 i=$((i+1))
 678
 679                 # let next job depend on previous
 680                 depend=$bid
 681         done
 682
 683         if [ "$post_command" ]; then
 684                 bid=$(uuidgen)
 685                 if [ "$no_depend" = yes ]; then
 686                         depend=
 687                 fi
 688                 echo -E >&2 "* Post-Command: $post_command"
 689                 echo "  <batch id='$bid' depends-on='$depend'>"
 690                 echo "           <command flags='$command_flags'>"
 691                 echo -E "$post_command" | sed -e 's,&,\&amp;,g;s,<,\&lt;,g;s,>,\&gt;,g'
 692                 echo "</command>"
 693                 echo "           <info></info>"
 694                 echo "$builders_xml"
 695                 echo "  </batch>"
 696                 depend=$bid
 697         fi
 698
 699         echo "</group>"
 700 }
 701
 702 gen_email () {
 703         # make request first, so the STDERR/STDOUT streams won't be mixed
 704         local req=$(gen_req)
 705
 706         if [ "$verbose" = "yes" ]; then
 707                 echo >&2 -E "$req"
 708         fi
 709 cat <<EOF
 710 From: $requester
 711 To: $builder_email
 712 Subject: build request
 713 Message-Id: <$id@$(hostname)>
 714 X-New-PLD-Builder: request
 715 X-Requester-Version: \$Id$
 716
 717 $(echo -E "$req" | gpg --clearsign --default-key $default_key $gpg_opts)
 718 EOF
 719 }
 720
 721 gen_email | send_request