bin/cleanbuild-vserver.sh

   1 #!/usr/bin/sudo /bin/sh
   2
   3 RPMS_FROM="$HOME/rpm/cleanRPMS.repo"
   4 DEST="th"
   5 SRC="-n th-x86_64-test"
   6 SUFFIX=""
   7 CACHEDIR="$PWD/poldekcache"
   8 RPMMACROS=""
   9 BUILDERRC=""
  10 IGNORE=""
  11 CHROOTSIZE="4G"
  12 ignore() { IGNORE="$IGNORE $*"; }
  13 NODEBUG=true
  14 CLEANAFTER=false
  15 FORCE_UMOUNT=false
  16 NOREBUILDDB=true
  17 MULTILIB=false
  18 # Whatever you set here as value, consider that it may not be shorter than
  19 # /usr/lib/debug (or /usr/src/debug) for debuginfo to work.
  20 # You get "Only dest dir longer than base dir not supported" error otherwise.
  21 BUILDDIR=/usr/src/BUILD
  22
  23 [ -r .cleanbuildrc ] && . ./.cleanbuildrc
  24
  25 [ -z "$USER" ] && echo "USER not defined" && exit 1
  26 [ "$USER" = "root" ] && echo "USER must not be root" && exit 1
  27
  28 export LC_ALL=C
  29 unset LANGUAGE
  30 unset LANG
  31
  32 usage() {
  33         [ $# -gt 0 ] && echo "$*"
  34         echo "Usage:"
  35         echo "  ./cleanbuild [cleanbuild options] specname [builder options]"
  36         echo "  ./build [cleanbuild options] specname [builder options]"
  37         echo "  ./clean [cleanbuild options]"
  38         echo "  ./create [cleanbuild options]"
  39         echo "  ./install [cleanbuild options] packages"
  40         echo ""
  41         echo "cleanbuild options:"
  42         echo " -32, -64, -th-i486  - select architecture"
  43         echo " --cleanafter | -ca  - clean after build"
  44         echo " --forceumount | -fu - force umount tmpfs"
  45         echo " --debug             - enable debug"
  46         echo " --network           - allow build to use networking"
  47         echo " -a, -b, -c, -d, -e  - select alternative chroot directory"
  48         exit 1
  49 }
  50
  51 FETCH=false
  52 CLEAN=false
  53 CREATE=false
  54 BUILD=false
  55 NETWORK=false
  56 INSTALL=false
  57
  58 case "$0" in
  59         *clean)
  60                 CLEAN=exit_after
  61                 ;;
  62         *cleanbuild)
  63                 FETCH=true
  64                 CLEAN=true
  65                 CREATE=true
  66                 BUILD=true
  67                 ;;
  68         *build)
  69                 CREATE=true
  70                 BUILD=true
  71                 ;;
  72         *create)
  73                 CLEAN=true
  74                 CREATE=exit_after
  75                 ;;
  76         *install)
  77                 CREATE=true
  78                 INSTALL=exit_after
  79                 ;;
  80         *)
  81                 usage
  82                 ;;
  83 esac
  84
  85 while [ $# -gt 0 ]; do
  86         OPT="${1}"
  87         case "$OPT" in
  88                 -32)    OPT="-th-i686" ;;
  89                 -64)    OPT="-th-x86_64" ;;
  90                 -th-32) OPT="-th-i686" ;;
  91                 -th-64) OPT="-th-x86_64" ;;
  92                 -ti-32) OPT="-ti-i686" ;;
  93                 -ti-64) OPT="-ti-x86_64" ;;
  94                 -ac)    OPT="-ac-amd64" ;;
  95                 -ac-32) OPT="-ac-i586" ;;
  96                 -ac-64) OPT="-ac-amd64" ;;
  97         esac
  98
  99         V="${OPT#-}"
 100         case "$OPT" in
 101                 -th-i[46]86 | -th-x86_64)
 102                         DEST="$V"
 103                         SRC="-n $V-ready"
 104                         ;;
 105                 -th-i[46]86-test | -th-x86_64-test)
 106                         DEST="$V"
 107                         SRC="-n $V"
 108                         ;;
 109                 -ti-i[56]86 | -ti-x86_64)
 110                         DEST="$V"
 111                         SRC="-n $V-ready"
 112                         ;;
 113                 -ti-i[56]86-test | -ti-x86_64-test)
 114                         DEST="$V"
 115                         SRC="-n $V"
 116                         ;;
 117                 -ac-amd64 | -ac-i[356]86 | -ac-athlon)
 118                         DEST="$V"
 119                         SRC="-n $V-ready"
 120                         ;;
 121                 -[1-9]G | -[1-9][0-9]G )
 122                         CHROOTSIZE="$V"
 123                         ;;
 124                 --cleanafter | -ca)
 125                         CLEANAFTER=true
 126                         ;;
 127                 --debug)
 128                         NODEBUG=false
 129                         ;;
 130                 --network)
 131                         NETWORK=true
 132                         ;;
 133                 --forceumount | -fu)
 134                         FORCE_UMOUNT=true
 135                         ;;
 136                 -[a-e])
 137                         SUFFIX="$OPT"
 138                         ;;
 139                 -~*)
 140                         SUFFIX="$V"
 141                         ;;
 142                 *)
 143                         break
 144                         ;;
 145         esac
 146         shift
 147 done
 148
 149 if $BUILD; then
 150         [ $# -ne 0 ] || usage
 151         build_pkg="${1}"
 152         build_pkg="${build_pkg%/}"
 153         build_pkg="${build_pkg%.spec}"
 154         build_pkg="${build_pkg#*/}"
 155         shift
 156
 157         builder_options="$*"
 158 fi
 159
 160 $NODEBUG || set -x
 161
 162 CHNAME="chroot-$DEST$SUFFIX"
 163 CHDIR="$PWD/$CHNAME"
 164 CHHOME="/home/users/$USER"
 165
 166 warn()
 167 {
 168         echo -n -e "\033[31;1m" >&2
 169         echo -n "$*" >&2
 170         echo -e "\033[0m" >&2
 171 }
 172
 173 die()
 174 {
 175         code=$1
 176         shift
 177         warn "$*"
 178         exit $code
 179 }
 180
 181 info()
 182 {
 183         echo -n -e "\033[32m"
 184         echo -n "$*"
 185         echo -e "\033[0m"
 186 }
 187
 188 title()
 189 {
 190         [ -t 1 ] || return 0
 191         local msg="$CHNAME: $build_pkg: $*"
 192         case "$TERM" in
 193                 cygwin|xterm*)
 194                         echo -ne "\033]1;$msg\007\033]2;$msg\007" >&2
 195                         ;;
 196                 screen*)
 197                         echo -ne "\033]0;$msg\007" >&2
 198                         ;;
 199         esac
 200         return 0
 201 }
 202
 203 exit_after()
 204 {
 205         return 0;
 206 }
 207
 208 check_running()
 209 {
 210         [ -r "$CHDIR/.pid" ] || return
 211         PID=$(< "$CHDIR/.pid")
 212         if [ -d /proc/$PID ]; then
 213                 die 10 "Another process ($PID) already running in $CHNAME"
 214         fi
 215 }
 216
 217 for D in installed buildlogs $CACHEDIR; do
 218         if [ ! -d "$D" ]; then
 219                 info "mkdir $D"
 220                 su $USER -c "mkdir -p $D" || die 13 "Cannot create work directories"
 221         fi
 222 done
 223
 224 ignore \
 225         upstart\* \
 226         upstart \
 227         compat-\* \
 228         systemd-init \
 229         hhvm \
 230         vserver-packages \
 231         xorg-driver-video-fglrx\* xorg-driver-video-nvidia\* xorg-xserver-xgl-libGL \
 232         xorg-driver-video-vboxvideo \
 233         xorg-data-xbitmaps \
 234         compat-gcc\* \
 235         libpng1\* \
 236         libjpeg libjpeg-devel \
 237         freetype1-devel-* \
 238         anacron fcron hc-cron \
 239         masqmail msmtp-sendmail omta postfix sendmail ssmtp nail-mail nullmailer \
 240         ghostscript-esp \
 241         perl-Scalar-List-Utils \
 242         perl-ExtUtils-Install \
 243         phonon-backend-mplayer phonon-backend-vlc \
 244         libgcj libgcj-devel \
 245         icedtea6-jre icedtea6-jdk \
 246         icedtea7-jre icedtea7-jdk \
 247         java-sun-jre java-sun-jdk \
 248         java5-sun-jre java5-sun-jdk \
 249         oracle-java7-jre oracle-java7-jdk \
 250         gnome-menus \
 251         gnome-speech-driver-festival gnome-speech-driver-speech-dispatcher
 252
 253 if ! $MULTILIB; then
 254         ignore '*-multilib-*'
 255 fi
 256
 257 rebuilddb()
 258 {
 259         $NOREBUILDDB || rpm --root=$CHDIR --rebuilddb
 260 }
 261
 262 poldek()
 263 {
 264         $NODEBUG || set -x
 265         rebuilddb
 266         /usr/bin/poldek $SRC -s "$RPMS_FROM" -r "$CHDIR" "--cachedir=$CACHEDIR" --conf=$PWD/poldekconf/poldek.conf "$@"
 267 }
 268
 269
 270 build_umount()
 271 {
 272         for DIR in $CHHOME/rpm $CHHOME dev proc sys; do
 273                 [ -d $CHDIR/$DIR ] && umount $CHDIR/$DIR
 274         done
 275 }
 276
 277 build_remove_root()
 278 {
 279         $NODEBUG || set -x
 280         if $FORCE_UMOUNT; then
 281                 # safety checks.
 282                 [ "$CHDIR" ] || exit 1
 283                 [ -d "$CHDIR" ] || exit 1
 284                 rm -rf $CHDIR/*
 285                 umount -l $CHDIR
 286         else
 287                 umount $CHDIR
 288         fi
 289         rmdir $CHDIR
 290 }
 291
 292 clean()
 293 {
 294         info "Cleaning $CHNAME"
 295         title "cleaning chroot"
 296         build_umount
 297         build_remove_root
 298 }
 299
 300 build_prepare_root()
 301 {
 302         title "preparing chroot"
 303         set -e
 304         $NODEBUG || set -x
 305         mkdir $CHDIR
 306         mount -t tmpfs -o size=$CHROOTSIZE,relatime /dev/null $CHDIR
 307         echo $$ > $CHDIR/.pid
 308
 309         rpmversion=$(rpm -E '%(v=%{_rpmversion}; IFS=.; set -- $v; echo $1)')
 310         rpmversion=${rpmversion:-4}
 311
 312         if [ "$rpmversion" -ge 5 ]; then
 313                 rpm --root=$CHDIR -qa
 314         else
 315                 rpm --root=$CHDIR --initdb
 316         fi
 317         poldek --up || :
 318         poldek -O "ignore=$IGNORE" -u rpm-build pwdutils coreutils time util-linux git-core gawk
 319         echo Poldek exit: $?
 320
 321         for DIR in dev proc sys; do
 322                 # We need to create these directories manually, because they are marked
 323                 # as netsharedpath in cleanbuild poldek.conf
 324                 mkdir $CHDIR/$DIR
 325                 mount -o bind /$DIR $CHDIR/$DIR
 326         done
 327
 328         # group 'users' may already exist, so ignore errors
 329         chroot $CHDIR groupadd $(id $USER -gn) -g$(id $USER -g) || :
 330         chroot $CHDIR useradd -m $USER -u$(id $USER -u) -g $(id $USER -gn)
 331
 332         # replicate files which already belong to $USER
 333         # so they will have correct owner and permissions
 334         cp -a $CHDIR/$CHHOME/{tmp,rpm}
 335         cp -a $CHDIR/$CHHOME/tmp $CHDIR$BUILDDIR
 336
 337         cp -a $CHDIR/$CHHOME/{.bashrc,.rpmmacros}
 338         cat <<-EOM > $CHDIR/$CHHOME/.rpmmacros
 339         %_builddir              $BUILDDIR
 340         %buildroot              %{_builddir}/%{name}-%{version}-root-%(id -u -n)
 341         %_rpmdirname    cleanRPMS
 342         %_rpmdir                %{expand:%%global _rpmdir %([ -d %{_topdir}/../%{_rpmdirname} ] && (cd %{_topdir}/../%{_rpmdirname}; pwd) || echo %{_topdir}/%{_rpmdirname})}%_rpmdir
 343         %distribution   CleanPLD
 344         %_binary_payload        w1.gzdio
 345 EOM
 346         [ -z "$RPMMACROS" ] || echo "$RPMMACROS" >> $CHDIR/$CHHOME/.rpmmacros
 347
 348         cp -a $CHDIR/$CHHOME/{.bashrc,.builderrc}
 349         cat <<-'EORC' > $CHDIR/$CHHOME/.builderrc
 350         TITLECHANGE=no
 351 EORC
 352         [ -z "$BUILDERRC" ] || echo "$BUILDERRC" >> $CHDIR/$CHHOME/.builderrc
 353
 354         set +e
 355 }
 356
 357 build_mount_home()
 358 {
 359         $NODEBUG || set -x
 360         mount -o bind $HOME/rpm $CHDIR/$CHHOME/rpm
 361
 362         # ensure RPMS dir is available
 363         chroot $CHDIR su $USER -c 'mkdir -p $(rpm -E %_rpmdir)'
 364 }
 365
 366 print_installed()
 367 {
 368         echo=$1
 369         if [ -r installed/$build_pkg ]; then
 370                 $echo "$(cat installed/$build_pkg | awk '{print $1}' | sort -u \
 371                         | awk '{br=br ", " $1} END{gsub(/^, /, "- BR: ", br ); print br}')"
 372                 cat installed/$build_pkg
 373         fi
 374 }
 375
 376 addlist()
 377 {
 378         LIST="$1"
 379
 380         print_installed info
 381
 382         return
 383         echo "*** $build_pkg $(date --rfc-3339=seconds) ***" >> $LIST
 384         print_installed echo >> $LIST
 385 }
 386
 387 builddie()
 388 {
 389         LIST="$1"; shift
 390         CODE="$1"; shift
 391         MSG="$*"
 392
 393         rm -f $CHDIR/.pid
 394
 395         $CLEANAFTER && clean
 396         title "failed !"
 397
 398         addlist "ERROR_$LIST"
 399         die $CODE "$MSG"
 400 }
 401
 402 LAST_INSTALL=""
 403 poldek_install()
 404 {
 405         local I="$1" ret
 406         # Nothing to install
 407         [ -n "$I" ] || return 1
 408         # Installing same packets second time
 409         [ "$LAST_INSTALL" != "$I" ] || return 1
 410         LAST_INSTALL="$I"
 411
 412         info "Installing" $I
 413         poldek -O "ignore=$IGNORE" -u $I | tee $$.poldek_install
 414         ret=
 415         if grep -q "Preparing...                ##################################################" $$.poldek_install \
 416                         && ! grep -q "file .* from install of .* conflicts with file from package" $$.poldek_install
 417                 then
 418                 info "Poldek:" $I "installed"
 419                 ret=0
 420         elif grep -q "Nothing to do" $$.poldek_install; then
 421                 warn "Poldek:" $I "installed already"
 422                 ret=1
 423         fi
 424         rm $$.poldek_install
 425         [ -n "$ret" ] && return $ret
 426
 427         # try harder
 428         info "Poldek install failed, retry without ignore"
 429         poldek -u $I && return 0
 430         info "Poldek install failed, retry once more without ignore"
 431         poldek -u $I && return 0
 432         warn "Poldek:" "Could not install" $I
 433         return 1
 434 }
 435
 436 maybe_call()
 437 {
 438         local cond="$1"; shift
 439         local func="$1"; shift
 440
 441         [ $cond = "false" ] && return
 442         "$func" "$@"
 443         [ $cond = "exit_after" ] && exit
 444 }
 445
 446 fetch()
 447 {
 448         info "Fetching $build_pkg"
 449         title "fetch"
 450         $NODEBUG || set -x
 451         su $USER -c "$HOME/rpm/packages/builder -g $build_pkg $builder_options" \
 452                 || die 11 "Fetch failed"
 453 }
 454
 455 create()
 456 {
 457         $NODEBUG || set -x
 458         su $USER -c "poldek -s $RPMS_FROM --mkidx"
 459
 460         if [ ! -d $CHDIR ]; then
 461                 info "Preparing $CHNAME"
 462                 build_prepare_root
 463                 build_mount_home
 464         fi
 465 }
 466
 467
 468 info "Configured Poldek sources"
 469 poldek -l
 470
 471 maybe_call $FETCH fetch
 472
 473 check_running
 474
 475 maybe_call $CLEAN clean
 476
 477 maybe_call $CREATE create
 478
 479 echo $$ > $CHDIR/.pid
 480
 481 maybe_call $INSTALL poldek_install "$*"
 482
 483 maybe_call $NETWORK cp -bf /etc/resolv.conf $CHDIR/etc/
 484
 485 $BUILD || exit
 486
 487 if [ -p /tmp/fixfreq ]; then
 488         echo $$ > /tmp/fixfreq
 489 fi
 490
 491 while true; do
 492         info "Building $build_pkg in $CHNAME"
 493         rebuilddb
 494         ./bin/cleanup-la $CHDIR
 495         buildlog="buildlogs/$build_pkg"
 496         if [ -r $buildlog ]; then
 497                 i=1
 498                 while [ -r $buildlog.$i ]; do
 499                         i=$((i+1))
 500                 done
 501                 info "moving $buildlog to $buildlog.$i"
 502                 mv $buildlog $buildlog.$i
 503         fi
 504         ./bin/findunusedbr -c $CHDIR $HOME/rpm/packages/$build_pkg/$build_pkg.spec
 505         title "building"
 506         ./bin/teeboth $buildlog chroot $CHDIR su $USER -c "$CHHOME/rpm/packages/builder -nn --define '_enable_debug_packages 0' -bb $build_pkg $builder_options"
 507         ECODE=$?
 508
 509         if grep -q "error: Failed build dependencies:" $buildlog; then
 510                 SEARCH=$(cat $buildlog | awk '/^Error:/ { p = 0 }; { if ( p ) { f="p"; if ( $1 ~ /^\// ) f="f"; printf "search -%c %s; ", f, $1; } }; /error: Failed build dependencies:/ { p = 1 }')
 511                 INSTALL=$(poldek -O "ignore=$IGNORE" --shcmd="$SEARCH" | awk '{ if ( p ) { print; p = 0; } } / package\(s\) found:$/ { p = 1 }' | sed 's/^\(.*\)-.*-.*$/\1/' | sort -u)
 512
 513                 if poldek_install "$INSTALL"; then
 514                         info "Deps installed"
 515                         continue
 516                 else
 517                         addlist ERROR_BRINSTALL
 518                         die 4 "Cannot install BRs"
 519                 fi
 520         fi
 521
 522         ./bin/findbr $CHDIR/$BUILDDIR $buildlog > $$.installed
 523         installed_something=false
 524         while read pkg msg; do
 525                 if poldek_install $pkg; then
 526                         info "findbr:" $pkg "installed"
 527                         echo "$pkg $msg" >> installed/$build_pkg
 528                         ./bin/addbr $build_pkg "$pkg" "$msg"
 529                         installed_something=true
 530                 else
 531                         warn "findbr:" $pkg "not installed"
 532                 fi
 533         done < $$.installed
 534         rm -f $$.installed
 535         $installed_something && continue
 536
 537         if [ $ECODE -eq 0 ]; then
 538                 $CLEANAFTER && clean
 539                 addlist BUILT_OK
 540                 ./bin/findunusedbr $CHDIR $HOME/rpm/packages/$build_pkg/$build_pkg.spec
 541                 info "$build_pkg built OK !"
 542                 title "OK !"
 543                 exit 0
 544         else
 545                 builddie UNKNOWN 1 "Got error but dunno what to do !"
 546         fi
 547 done
 548
 549
 550 # vim: ts=4 sw=4 filetype=sh