client/make-request.sh

   1 #!/bin/sh
   2
   3 # prevent "*" from being expanded in builders var
   4 set -f
   5
   6 builders=
   7 with=
   8 without=
   9 flags=
  10 command=
  11 command_flags=
  12 gpg_opts=
  13 default_branch='HEAD'
  14 distro=
  15 url=
  16 no_depend=no
  17 verbose=no
  18
  19 if [ -x /usr/bin/python ]; then
  20         send_mode="python"
  21 else
  22         echo "No python present, using mail mode"
  23         send_mode="mail"
  24 fi
  25
  26 if [ -n "$HOME_ETC" ]; then
  27         USER_CFG=$HOME_ETC/.requestrc
  28 else
  29         USER_CFG=$HOME/.requestrc
  30 fi
  31
  32 if [ ! -f "$USER_CFG" ]; then
  33         echo "Creating config file $USER_CFG. You *must* edit it."
  34         cat > $USER_CFG <<EOF
  35 priority=2
  36 requester=deviloper@pld-linux.org
  37 default_key=deviloper@pld-linux.org
  38 send_mode="$send_mode"
  39 url="$url"
  40 mailer="/usr/sbin/sendmail -t"
  41 gpg_opts=""
  42 distro=th
  43 url="http://src.th.pld-linux.org:1234/"
  44
  45 # defaults:
  46 f_upgrade=yes
  47 EOF
  48 exit
  49 fi
  50
  51 if [ -f "$USER_CFG" ]; then
  52         . $USER_CFG
  53 fi
  54
  55 # internal options, not to be overriden
  56 specs=
  57 df_fetch=no
  58 upgrade_macros=no
  59
  60 # Set colors
  61 c_star=$(tput setaf 2)
  62 c_red=$(tput setaf 1)
  63 c_norm=$(tput op)
  64 msg() {
  65         echo >&2 "${c_star}*${c_norm} $*"
  66 }
  67 red() {
  68         echo "${c_red}$*${c_norm}"
  69 }
  70
  71 die() {
  72         echo >&2 "$0: $*"
  73         exit 1
  74 }
  75
  76 send_request() {
  77         # switch to mail mode, if no url set
  78         [ -z "$url" ] && send_mode="mail"
  79
  80         case "$send_mode" in
  81         "mail")
  82                 msg "Sending using mail mode"
  83                 cat - | $mailer
  84                 ;;
  85         *)
  86                 msg "Sending using http mode to $url"
  87                 cat - | python -c '
  88 import sys, socket, urllib2
  89
  90 try:
  91         data = sys.stdin.read()
  92         url = sys.argv[1]
  93         socket.setdefaulttimeout(10)
  94         req = urllib2.Request(url, data)
  95         f = urllib2.urlopen(req)
  96         f.close()
  97 except Exception, e:
  98         print >> sys.stderr, "Problem while sending request via HTTP: %s: %s" % (url, e)
  99         sys.exit(1)
 100 print >> sys.stdout, "Request queued via HTTP."
 101 ' "$url"
 102                 ;;
 103         esac
 104 }
 105
 106 # simple df_fetcher, based on packages/fetchsrc_request
 107 # TODO: tcp (smtp) mode
 108 # TODO: adjust for ~/.requestrc config
 109 df_fetch() {
 110         local specs="$@"
 111
 112         # Sending by
 113         local MAILER='/usr/sbin/sendmail'
 114         # MAILER='/usr/bin/msmtp'
 115         # Sending via
 116         local VIA="SENDMAIL"
 117         #VIA="localhost"
 118         local VIA_ARGS=""
 119         #VIA_ARGS="some additional flags"
 120         # e.g. for msmtp:
 121         # VIA_ARGS='-a gmail'
 122         #
 123         # DISTFILES EMAIL
 124         local DMAIL="distfiles@pld-linux.org"
 125
 126         local HOST=$(hostname -f)
 127         local LOGIN=${requester%@*}
 128
 129         for spec in $specs; do
 130                 local SPEC=$(echo "$spec" | sed -e 's|:.*||')
 131                 local BRANCH=$(echo "$spec" | sed -e 's|.*:||')
 132                 echo >&2 "Distfiles Request: $SPEC:$BRANCH via $MAILER ${VIA_ARGS:+ ($VIA_ARGS)}"
 133                 cat <<-EOF | "$MAILER" -t -i $VIA_ARGS
 134                         To: $DMAIL
 135                         From: $LOGIN <$LOGIN@$HOST>
 136                         Subject: fetchsrc_request notify
 137                         X-CVS-Module: SPECS
 138                         X-distfiles-request: yes
 139                         X-Login: $LOGIN
 140                         X-Spec: $SPEC
 141                         X-Branch: $BRANCH
 142                         X-Flags: force-reply
 143
 144                         .
 145                         EOF
 146         done
 147 }
 148
 149 usage() {
 150         cat <<EOF
 151 Usage: make-request.sh [OPTION] ... [SPECFILE] ....
 152
 153 Mandatory arguments to long options are mandatory for short options too.
 154
 155       --config-file /path/to/config/file
 156             Source additional config file (after $USER_CFG), useful when
 157             when sending build requests to Ac/Th from the same account
 158       -b 'BUILDER BUILDER ...',  --builder='BUILDER BUILDER ...'
 159            Sends request to given builders (in 'version-arch' format)
 160       --with VALUE, --without VALUE
 161             Build package with(out) a given bcond
 162       --kernel VALUE
 163             set alt_kernel to VALUE
 164       --target VALUE
 165             set --target to VALUE
 166       -s BUILD_ID, --skip BUILD_ID[,BUILD_ID][,BUILD_ID]
 167             mark build ids on src builder to be skipped
 168       --branch VALUE
 169             specify default branch for specs in request
 170       -t, --test-build
 171             Performs a 'test-build'. Package will be uploaded to hidden .test-builds/
 172             ftp tree and won't be upgraded on builders.
 173       -r, --ready-build
 174             Preforms a 'ready' build. Package will be built and uploaded to test/ ftp tree
 175             (and later moved by release manager staff to ready/ and main ftp tree)
 176       -u, --upgrade
 177             Forces package upgrade (for use with -c or -q, not -t)
 178       -n, --no-upgrade
 179             Disables package upgrade (for use with -r)
 180       -ni, --no-install-br
 181             Do not install missing BuildRequires (--nodeps)
 182       -nd, --no-depend
 183             Do not add dependency of build jobs, each job in batch runs itself
 184       -j, --jobs
 185             Number of parallel jobs for single build
 186       -f, --flag
 187       -d, --distro DISTRO
 188             Specify value for \$distro
 189       -df,  --distfiles-fetch[-request] PACKAGE
 190             Send distfiles request to fetch sources for PACKAGE
 191       -cf, --command-flag
 192             Not yet documented
 193       -c, --command
 194             Executes a given command on builders (prepended to build jobs if build jobs included)
 195       -C, --post-command
 196             Executes a given command on builders (appended to build jobs if build jobs included)
 197       --test-remove-pkg
 198             shortcut for --command poldek -evt ARGS
 199       --remove-pkg
 200             shortcut for --command poldek -ev --noask ARGS
 201       --upgrade-pkg
 202             shortcut for --command poldek --up -Uv ARGS
 203       --pull
 204             Updates builders infrastructure (outside chroot)
 205       --update-macros
 206             Updates rpm-build-macros on src builder
 207       -q
 208             shortcut for --command rpm -q ARGS
 209       -g, --gpg-opts "opts"
 210             Pass additional options to gpg binary
 211       -p, --priority VALUE
 212             sets request priority (default 2)
 213       -h, --help
 214             Displays this help message
 215 EOF
 216         exit 0
 217 }
 218
 219
 220 while [ $# -gt 0 ] ; do
 221         case "$1" in
 222                 --distro | -d)
 223                         distro=$2
 224                         shift
 225                         ;;
 226
 227                 --config-file)
 228                         [ -f "$2" ] && . $2 || die "Config file not found"
 229                         shift
 230                         ;;
 231
 232                 --builder | -b)
 233                         for b in $2; do
 234                                 builders="$builders ${b%:*}"
 235                         done
 236                         shift
 237                         ;;
 238
 239                 --with)
 240                         with="$with $(echo "$2" | tr ',' ' ')"
 241                         shift
 242                         ;;
 243
 244                 --without)
 245                         without="$without $(echo "$2" | tr ',' ' ')"
 246                         shift
 247                         ;;
 248
 249                 --test-build | -t)
 250                         build_mode=test
 251                         f_upgrade=no
 252                         ;;
 253
 254                 --kernel)
 255                         kernel=$2
 256                         shift
 257                         ;;
 258
 259                 --target)
 260                         target=$2
 261                         shift
 262                         ;;
 263
 264                 -s|--skip)
 265                         skip="$2"
 266                         shift
 267                         ;;
 268
 269                 --branch)
 270                         branch=$2
 271                         shift
 272                         ;;
 273
 274                 --priority | -p)
 275                         priority=$2
 276                         shift
 277                         ;;
 278
 279                 --ready-build | -r)
 280                         build_mode=ready
 281                         ;;
 282
 283                 --upgrade | -u)
 284                         f_upgrade=yes
 285                         ;;
 286
 287                 --no-upgrade | -n)
 288                         f_upgrade=no
 289                         ;;
 290
 291                 --no-depend | -nd)
 292                         no_depend=yes
 293                         ;;
 294
 295                 --no-install-br | -ni)
 296                         flags="$flags no-install-br"
 297                         ;;
 298
 299                 -j | --jobs)
 300                         jobs="$2"
 301                         shift
 302                         ;;
 303
 304                 -j*)
 305                         jobs="${1#-j}"
 306                         ;;
 307
 308                 -v)
 309                         verbose=yes
 310                         ;;
 311
 312                 --flag | -f)
 313                         flags="$flags $2"
 314                         shift
 315                         ;;
 316
 317                 --command-flags | -cf)
 318                         command_flags="$2"
 319                         shift
 320                         ;;
 321
 322                 --command | -c)
 323                         command="$2"
 324                         if [ "$command" = - ]; then
 325                                 echo >&2 "Reading command from STDIN"
 326                                 echo >&2 "---"
 327                                 command=$(cat)
 328                                 echo >&2 "---"
 329                         fi
 330                         shift
 331                         ;;
 332                 --post-command | -C)
 333                         post_command="$2"
 334                         if [ "$post_command" = - ]; then
 335                                 echo >&2 "Reading post_command from STDIN"
 336                                 echo >&2 "---"
 337                                 post_command=$(cat)
 338                                 echo >&2 "---"
 339                         fi
 340                         shift
 341                         ;;
 342                 --test-remove-pkg)
 343                         command="poldek -evt $2"
 344                         f_upgrade=no
 345                         shift
 346                         ;;
 347                 --remove-pkg)
 348                         command="for a in $2; do poldek -ev --noask \$a; done"
 349                         f_upgrade=no
 350                         shift
 351                         ;;
 352                 --upgrade-pkg|-Uhv)
 353                         command="poldek --up; poldek -uv $2"
 354                         f_upgrade=no
 355                         shift
 356                         ;;
 357                 -q)
 358                         command="rpm -q $2"
 359                         f_upgrade=no
 360                         shift
 361                         ;;
 362
 363                 --pull)
 364                         command_flags="no-chroot"
 365                         command="git pull"
 366                         f_upgrade=no
 367                         ;;
 368
 369                 --update-macros)
 370                         upgrade_macros="yes"
 371                         ;;
 372
 373                 -df | --distfiles-fetch | --distfiles-fetch-request)
 374                         df_fetch=yes
 375                         ;;
 376
 377                 --gpg-opts | -g)
 378                         gpg_opts="$2"
 379                         shift
 380                         ;;
 381
 382                 --help | -h)
 383                         usage
 384                         ;;
 385
 386                 -*)
 387                         die "unknown knob: $1"
 388                         ;;
 389
 390                 *:* | *)
 391                         specs="$specs $1"
 392                         ;;
 393         esac
 394         shift
 395 done
 396
 397 case "$distro" in
 398 ac)
 399         builder_email="builder-ac@pld-linux.org"
 400         default_builders="ac-*"
 401         default_branch="AC-branch"
 402         url="http://ep09.pld-linux.org:1289/"
 403         control_url="http://ep09.pld-linux.org/~buildsrc"
 404         ;;
 405 ac-java) # fake "distro" for java available ac architectures
 406         builder_email="builder-ac@pld-linux.org"
 407         default_builders="ac-i586 ac-i686 ac-athlon ac-amd64"
 408         default_branch="AC-branch"
 409         url="http://ep09.pld-linux.org:1289/"
 410         ;;
 411 ac-xen) # fake "distro" for xen-enabled architectures
 412         builder_email="builder-ac@pld-linux.org"
 413         default_builders="ac-i686 ac-athlon ac-amd64"
 414         default_branch="AC-branch"
 415         ;;
 416 ti)
 417         builder_email="builderti@ep09.pld-linux.org"
 418         default_builders="ti-*"
 419         url="http://ep09.pld-linux.org:1231/"
 420         control_url="http://ep09.pld-linux.org/~builderti"
 421         ;;
 422 ti-dev)
 423         builder_email="buildertidev@ep09.pld-linux.org"
 424         default_builders="ti-dev-*"
 425         url="http://ep09.pld-linux.org:1232/"
 426         control_url="http://ep09.pld-linux.org/~buildertidev"
 427         ;;
 428 th)
 429         builder_email="builderth@pld-linux.org"
 430         default_builders="th-*"
 431         url="http://src.th.pld-linux.org:1234/"
 432         control_url="http://src.th.pld-linux.org"
 433         ;;
 434 th-java) # fake "distro" for java available th architectures
 435         builder_email="builderth@pld-linux.org"
 436         default_builders="th-x86_64 th-athlon th-i686"
 437         url="http://src.th.pld-linux.org:1234/"
 438         ;;
 439 aidath)
 440         builder_email="builderaidath@ep09.pld-linux.org"
 441         default_builders="aidath-*"
 442         ;;
 443 *)
 444         die "distro \`$distro' not known"
 445         ;;
 446 esac
 447
 448 # need to do this after distro selection
 449 if [ "$skip" ]; then
 450         skip=$(skip="$skip" control_url="$control_url" python -c '
 451 import urllib2
 452 import sys
 453 import StringIO
 454 import gzip
 455 import re
 456 import os
 457 import string
 458 from xml.dom import minidom
 459
 460 skip = os.environ.get("skip").split(",");
 461 control_url = os.environ.get("control_url")
 462
 463 print >> sys.stderr, "* Check queue_id-s against %s" % control_url
 464
 465 try:
 466         headers = { "Cache-Control": "no-cache", "Pragma": "no-cache" }
 467         req = urllib2.Request(url=control_url + "/queue.gz", headers=headers)
 468         f = urllib2.urlopen(req)
 469 except Exception, e:
 470         print >> sys.stderr, "Fetch error %s: %s" % (control_url + "/queue.gz", e)
 471         sys.exit(1)
 472
 473 sio = StringIO.StringIO()
 474 sio.write(f.read())
 475 f.close()
 476 sio.seek(0)
 477 f = gzip.GzipFile(fileobj = sio)
 478
 479 xml = re.compile("(<queue>.*?</queue>)", re.DOTALL).match(f.read()).group(1)
 480 d = minidom.parseString(xml)
 481
 482 q = []
 483 for c in d.documentElement.childNodes:
 484         if c.nodeName != "group":
 485                 continue
 486         q.append(c.attributes["id"].value)
 487
 488 err = 0
 489 for s in skip:
 490         if s not in q:
 491                 print >> sys.stderr, "- Check %s: ERROR: Not valid queue-id" % s
 492                 err = 1
 493         else:
 494                 print >> sys.stderr, "- Check %s: OK" % s
 495 if err == 1:
 496         sys.exit(1)
 497 print string.join(skip, ",")
 498 ') || exit $?
 499         f_upgrade=no
 500         build_mode=test
 501         priority=-1
 502         command="skip:$skip"
 503         command_flags="no-chroot"
 504         builders="$distro-src"
 505 fi
 506
 507 branch=${branch:-$default_branch}
 508
 509 specs=`for s in $specs; do
 510         case "$s" in
 511         ^)
 512                 # skip marker
 513                 echo $s
 514                 ;;
 515         *.spec:*) # spec with branch
 516                 basename $s
 517                 ;;
 518         *.spec) # spec without branch
 519                 echo $(basename $s):$branch
 520                 ;;
 521         *:*) # package name with branch
 522                 basename $s | sed -e 's/:/.spec:/'
 523                 ;;
 524         *) # just package name
 525                 echo $(basename $s).spec:$branch
 526                 ;;
 527         esac
 528 done`
 529
 530 if [ "$df_fetch" = "yes" ]; then
 531         df_fetch $specs
 532         exit 0
 533 fi
 534
 535 if [ "$upgrade_macros" = "yes" ]; then
 536         command="poldek --up; poldek -uv rpm-build-macros"
 537         builders="$distro-src"
 538         f_upgrade=no
 539         build_mode=test
 540 fi
 541
 542 if [[ "$requester" != *@* ]] ; then
 543         requester="$requester@pld-linux.org"
 544 fi
 545
 546 if [ -z "$builders" ] ; then
 547         builders="$default_builders"
 548 fi
 549
 550 if [ "$f_upgrade" = "yes" ] ; then
 551         flags="$flags upgrade"
 552 fi
 553
 554 if [ "$build_mode" = "test" ] ; then
 555         if [ "$f_upgrade" = "yes" ] ; then
 556                 die "--upgrade and --test-build are mutually exclusive"
 557         fi
 558         flags="$flags test-build"
 559 fi
 560
 561 if [ -z "$build_mode" ] ; then
 562         # missing build mode, builders go crazy when you proceed"
 563         die "please specify build mode"
 564 fi
 565
 566
 567 ok=
 568 for s in $specs; do
 569         ok=1
 570 done
 571
 572 if [ -z "$specs" -a -z "$command" ]; then
 573         die "no packages to build or command to invoke specified"
 574 fi
 575
 576 id=$(uuidgen)
 577
 578 gen_req() {
 579         echo "<group id='$id' no='0' flags='$flags'>"
 580         echo "  <time>$(date +%s)</time>"
 581         msg "Using priority $priority"
 582         echo "  <priority>$priority</priority>"
 583         if [ -n "$jobs" ]; then
 584                 msg "Using jobs $jobs"
 585                 echo "  <maxjobs>$jobs</maxjobs>"
 586         fi
 587         msg "Build mode: $build_mode"
 588         if [ -z "$url" ]; then
 589                 msg "Using email $builder_email"
 590         else
 591                 msg "Using URL $url"
 592         fi
 593         msg "Queue-ID: $id"
 594         echo
 595
 596         # job to depend on
 597         local depend=
 598         local b i=1
 599         local name branch builders_xml
 600
 601         for b in $builders; do
 602                 msg "Builder: $(red $b)"
 603                 builders_xml="$builders_xml <builder>$b</builder>"
 604         done
 605
 606         if [ "$command" ]; then
 607                 bid=$(uuidgen)
 608                 echo -E >&2 "* Command: $command"
 609                 echo "  <batch id='$bid' depends-on=''>"
 610                 echo "           <command flags='$command_flags'>"
 611                 echo -E "$command" | sed -e 's,&,\&amp;,g;s,<,\&lt;,g;s,>,\&gt;,g'
 612                 echo "</command>"
 613                 echo "           <info></info>"
 614                 echo "$builders_xml"
 615                 echo "  </batch>"
 616                 depend=$bid
 617         fi
 618
 619         if [ "$f_upgrade" = "yes" ] ; then
 620                 msg "Upgrade mode: $f_upgrade"
 621         fi
 622
 623         for s in $specs; do
 624                 # skip marker
 625                 if [ "$s" = "^" ]; then
 626                         depend=
 627                         continue
 628                 fi
 629                 if [ "$no_depend" = yes ]; then
 630                         depend=
 631                 fi
 632                 bid=$(uuidgen)
 633                 echo "  <batch id='$bid' depends-on='$depend'>"
 634
 635                 name=$(echo "$s" | sed -e 's|:.*||')
 636                 branch=$(echo "$s" | sed -e 's|.*:||')
 637                 msg "Adding #$i $name:$branch${kernel:+ alt_kernel=$kernel}${target:+ target=$target}${depend:+ depends on $depend}"
 638                 echo "           <spec>$name</spec>"
 639                 echo "           <branch>$branch</branch>"
 640                 echo "           ${kernel:+<kernel>$kernel</kernel>}"
 641                 echo "           ${target:+<target>$target</target>}"
 642                 echo "           <info></info>"
 643                 echo
 644                 for b in $with; do
 645                         echo "           <with>$b</with>"
 646                 done
 647                 for b in $without; do
 648                         echo "           <without>$b</without>"
 649                 done
 650                 echo
 651                 echo "$builders_xml"
 652                 echo "  </batch>"
 653                 i=$((i+1))
 654
 655                 # let next job depend on previous
 656                 depend=$bid
 657         done
 658
 659         if [ "$post_command" ]; then
 660                 bid=$(uuidgen)
 661                 if [ "$no_depend" = yes ]; then
 662                         depend=
 663                 fi
 664                 echo -E >&2 "* Post-Command: $post_command"
 665                 echo "  <batch id='$bid' depends-on='$depend'>"
 666                 echo "           <command flags='$command_flags'>"
 667                 echo -E "$post_command" | sed -e 's,&,\&amp;,g;s,<,\&lt;,g;s,>,\&gt;,g'
 668                 echo "</command>"
 669                 echo "           <info></info>"
 670                 echo "$builders_xml"
 671                 echo "  </batch>"
 672                 depend=$bid
 673         fi
 674
 675         echo "</group>"
 676 }
 677
 678 gen_email () {
 679         # make request first, so the STDERR/STDOUT streams won't be mixed
 680         local req=$(gen_req)
 681
 682         if [ "$verbose" = "yes" ]; then
 683                 echo >&2 -E "$req"
 684         fi
 685 cat <<EOF
 686 From: $requester
 687 To: $builder_email
 688 Subject: build request
 689 Message-Id: <$id@$(hostname)>
 690 X-New-PLD-Builder: request
 691 X-Requester-Version: \$Id$
 692
 693 $(echo -E "$req" | gpg --clearsign --default-key $default_key $gpg_opts)
 694 EOF
 695 }
 696
 697 gen_email | send_request