cleanbuild

   1 #!/usr/bin/sudo /bin/sh
   2
   3 RPMS_FROM="$HOME/rpm/cleanRPMS.repo"
   4 DEST="th"
   5 SRC="-n th-x86_64-test"
   6 SUFFIX=""
   7 CACHEDIR="$PWD/poldekcache"
   8 RPMMACROS=""
   9 BUILDERRC=""
  10 IGNORE=""
  11 CHROOTSIZE="4G"
  12 ignore() { IGNORE="$IGNORE $*"; }
  13 NODEBUG=true
  14 CLEANAFTER=false
  15 FORCE_UMOUNT=false
  16 NOREBUILDDB=true
  17 MULTILIB=false
  18 # Whatever you set here as value, consider that it may not be shorter than
  19 # /usr/lib/debug (or /usr/src/debug) for debuginfo to work.
  20 # You get "Only dest dir longer than base dir not supported" error otherwise.
  21 BUILDDIR=/usr/src/BUILD
  22
  23 [ -r .cleanbuildrc ] && . ./.cleanbuildrc
  24
  25 [ -z "$USER" ] && echo "USER not defined" && exit 1
  26 [ "$USER" = "root" ] && echo "USER must not be root" && exit 1
  27
  28 export LC_ALL=C
  29 unset LANGUAGE
  30 unset LANG
  31
  32 usage() {
  33         [ $# -gt 0 ] && echo "$*"
  34         echo "Usage:"
  35         echo "  ./cleanbuild [cleanbuild options] specname [builder options]"
  36         echo "  ./build [cleanbuild options] specname [builder options]"
  37         echo "  ./clean [cleanbuild options]"
  38         echo "  ./create [cleanbuild options]"
  39         echo "  ./install [cleanbuild options] packages"
  40         echo ""
  41         echo "cleanbuild options:"
  42         echo " -32, -64, -th-i486  - select architecture"
  43         echo " --cleanafter | -ca  - clean after build"
  44         echo " --forceumount | -fu - force umount tmpfs"
  45         echo " --debug             - enable debug"
  46         echo " --network           - allow build to use networking"
  47         echo " -a, -b, -c, -d, -e  - select alternative chroot directory"
  48         exit 1
  49 }
  50
  51 FETCH=false
  52 CLEAN=false
  53 CREATE=false
  54 BUILD=false
  55 NETWORK=false
  56 INSTALL=false
  57
  58 case "$0" in
  59         *clean)
  60                 CLEAN=exit_after
  61                 ;;
  62         *cleanbuild)
  63                 FETCH=true
  64                 CLEAN=true
  65                 CREATE=true
  66                 BUILD=true
  67                 ;;
  68         *build)
  69                 CREATE=true
  70                 BUILD=true
  71                 ;;
  72         *create)
  73                 CLEAN=true
  74                 CREATE=exit_after
  75                 ;;
  76         *install)
  77                 CREATE=true
  78                 INSTALL=exit_after
  79                 ;;
  80         *)
  81                 usage
  82                 ;;
  83 esac
  84
  85 while [ $# -gt 0 ]; do
  86         OPT="${1}"
  87         case "$OPT" in
  88                 -32)    OPT="-th-i686" ;;
  89                 -64)    OPT="-th-x86_64" ;;
  90                 -th-32) OPT="-th-i686" ;;
  91                 -th-64) OPT="-th-x86_64" ;;
  92                 -ti-32) OPT="-ti-i686" ;;
  93                 -ti-64) OPT="-ti-x86_64" ;;
  94                 -ac)    OPT="-ac-amd64" ;;
  95                 -ac-32) OPT="-ac-i586" ;;
  96                 -ac-64) OPT="-ac-amd64" ;;
  97         esac
  98
  99         V="${OPT#-}"
 100         case "$OPT" in
 101                 -th-i[46]86 | -th-x86_64)
 102                         DEST="$V"
 103                         SRC="-n $V-ready"
 104                         ;;
 105                 -th-i[46]86-test | -th-x86_64-test)
 106                         DEST="$V"
 107                         SRC="-n $V"
 108                         ;;
 109                 -ti-i[56]86 | -ti-x86_64)
 110                         DEST="$V"
 111                         SRC="-n $V-ready"
 112                         ;;
 113                 -ti-i[56]86-test | -ti-x86_64-test)
 114                         DEST="$V"
 115                         SRC="-n $V"
 116                         ;;
 117                 -ac-amd64 | -ac-i[356]86 | -ac-athlon)
 118                         DEST="$V"
 119                         SRC="-n $V-ready"
 120                         ;;
 121                 -[1-9]G | -[1-9][0-9]G )
 122                         CHROOTSIZE="$V"
 123                         ;;
 124                 --cleanafter | -ca)
 125                         CLEANAFTER=true
 126                         ;;
 127                 --debug)
 128                         NODEBUG=false
 129                         ;;
 130                 --network)
 131                         NETWORK=true
 132                         ;;
 133                 --forceumount | -fu)
 134                         FORCE_UMOUNT=true
 135                         ;;
 136                 -[a-e])
 137                         SUFFIX="$OPT"
 138                         ;;
 139                 -~*)
 140                         SUFFIX="$V"
 141                         ;;
 142                 *)
 143                         break
 144                         ;;
 145         esac
 146         shift
 147 done
 148
 149 if $BUILD; then
 150         [ $# -ne 0 ] || usage
 151         build_pkg="${1}"
 152         build_pkg="${build_pkg%/}"
 153         build_pkg="${build_pkg%.spec}"
 154         build_pkg="${build_pkg#*/}"
 155         shift
 156
 157         builder_options="$*"
 158 fi
 159
 160 $NODEBUG || set -x
 161
 162 CHNAME="chroot-$DEST$SUFFIX"
 163 CHDIR="$PWD/$CHNAME"
 164 CHHOME="/home/users/$USER"
 165
 166 warn()
 167 {
 168         echo -n -e "\033[31;1m" >&2
 169         echo -n "$*" >&2
 170         echo -e "\033[0m" >&2
 171 }
 172
 173 die()
 174 {
 175         code=$1
 176         shift
 177         warn "$*"
 178         exit $code
 179 }
 180
 181 info()
 182 {
 183         echo -n -e "\033[32m"
 184         echo -n "$*"
 185         echo -e "\033[0m"
 186 }
 187
 188 title()
 189 {
 190         [ -t 1 ] || return 0
 191         local msg="$CHNAME: $build_pkg: $*"
 192         case "$TERM" in
 193                 cygwin|xterm*)
 194                         echo -ne "\033]1;$msg\007\033]2;$msg\007" >&2
 195                         ;;
 196                 screen*)
 197                         echo -ne "\033]0;$msg\007" >&2
 198                         ;;
 199         esac
 200         return 0
 201 }
 202
 203 exit_after()
 204 {
 205         return 0;
 206 }
 207
 208 check_running()
 209 {
 210         [ -r "$CHDIR/.pid" ] || return
 211         PID=$(< "$CHDIR/.pid")
 212         if [ -d /proc/$PID ]; then
 213                 die 10 "Another process ($PID) already running in $CHNAME"
 214         fi
 215 }
 216
 217 for D in installed buildlogs $CACHEDIR; do
 218         if [ ! -d "$D" ]; then
 219                 info "mkdir $D"
 220                 su $USER -c "mkdir -p $D" || die 13 "Cannot create work directories"
 221         fi
 222 done
 223
 224 ignore \
 225         upstart\* \
 226         upstart \
 227         compat-\* \
 228         systemd-init \
 229         hhvm \
 230         vserver-packages \
 231         xorg-driver-video-fglrx\* xorg-driver-video-nvidia\* xorg-xserver-xgl-libGL \
 232         xorg-driver-video-vboxvideo \
 233         xorg-data-xbitmaps \
 234         compat-gcc\* \
 235         libpng1\* \
 236         libjpeg libjpeg-devel \
 237         freetype1-devel-* \
 238         anacron fcron hc-cron \
 239         masqmail msmtp-sendmail omta postfix sendmail ssmtp nail-mail nullmailer \
 240         ghostscript-esp \
 241         perl-Scalar-List-Utils \
 242         perl-ExtUtils-Install \
 243         phonon-backend-mplayer phonon-backend-vlc \
 244         icedtea6-jre icedtea6-jdk \
 245         java-sun-jre java5-sun-jre oracle-java7-jre \
 246         gnome-menus \
 247         gnome-speech-driver-festival gnome-speech-driver-speech-dispatcher
 248
 249 if ! $MULTILIB; then
 250         ignore '*-multilib-*'
 251 fi
 252
 253 rebuilddb()
 254 {
 255         $NOREBUILDDB || rpm --root=$CHDIR --rebuilddb
 256 }
 257
 258 poldek()
 259 {
 260         $NODEBUG || set -x
 261         rebuilddb
 262         /usr/bin/poldek $SRC -s "$RPMS_FROM" -r "$CHDIR" "--cachedir=$CACHEDIR" --conf=$PWD/poldekconf/poldek.conf "$@"
 263 }
 264
 265
 266 build_umount()
 267 {
 268         for DIR in $CHHOME/rpm $CHHOME dev proc sys; do
 269                 [ -d $CHDIR/$DIR ] && umount $CHDIR/$DIR
 270         done
 271 }
 272
 273 build_remove_root()
 274 {
 275         $NODEBUG || set -x
 276         if $FORCE_UMOUNT; then
 277                 # safety checks.
 278                 [ "$CHDIR" ] || exit 1
 279                 [ -d "$CHDIR" ] || exit 1
 280                 rm -rf $CHDIR/*
 281                 umount -l $CHDIR
 282         else
 283                 umount $CHDIR
 284         fi
 285         rmdir $CHDIR
 286 }
 287
 288 clean()
 289 {
 290         info "Cleaning $CHNAME"
 291         title "cleaning chroot"
 292         build_umount
 293         build_remove_root
 294 }
 295
 296 build_prepare_root()
 297 {
 298         title "preparing chroot"
 299         set -e
 300         $NODEBUG || set -x
 301         mkdir $CHDIR
 302         mount -t tmpfs -o size=$CHROOTSIZE,relatime /dev/null $CHDIR
 303         echo $$ > $CHDIR/.pid
 304
 305         rpmversion=$(rpm -E '%(v=%{_rpmversion}; IFS=.; set -- $v; echo $1)')
 306         rpmversion=${rpmversion:-4}
 307
 308         if [ "$rpmversion" -ge 5 ]; then
 309                 rpm --root=$CHDIR -qa
 310         else
 311                 rpm --root=$CHDIR --initdb
 312         fi
 313         poldek --up || :
 314         poldek -O "ignore=$IGNORE" -u rpm-build pwdutils coreutils time util-linux git-core gawk
 315         echo Poldek exit: $?
 316
 317         for DIR in dev proc sys; do
 318                 # We need to create these directories manually, because they are marked
 319                 # as netsharedpath in cleanbuild poldek.conf
 320                 mkdir $CHDIR/$DIR
 321                 mount -o bind /$DIR $CHDIR/$DIR
 322         done
 323
 324         # group 'users' may already exist, so ignore errors
 325         chroot $CHDIR groupadd $(id $USER -gn) -g$(id $USER -g) || :
 326         chroot $CHDIR useradd -m $USER -u$(id $USER -u) -g $(id $USER -gn)
 327
 328         # replicate files which already belong to $USER
 329         # so they will have correct owner and permissions
 330         cp -a $CHDIR/$CHHOME/{tmp,rpm}
 331         cp -a $CHDIR/$CHHOME/tmp $CHDIR$BUILDDIR
 332
 333         cp -a $CHDIR/$CHHOME/{.bashrc,.rpmmacros}
 334         cat <<-EOM > $CHDIR/$CHHOME/.rpmmacros
 335         %_builddir              $BUILDDIR
 336         %buildroot              %{_builddir}/%{name}-%{version}-root-%(id -u -n)
 337         %_rpmdirname    cleanRPMS
 338         %_rpmdir                %{expand:%%global _rpmdir %([ -d %{_topdir}/../%{_rpmdirname} ] && (cd %{_topdir}/../%{_rpmdirname}; pwd) || echo %{_topdir}/%{_rpmdirname})}%_rpmdir
 339         %distribution   CleanPLD
 340         %_binary_payload        w1.gzdio
 341 EOM
 342         [ -z "$RPMMACROS" ] || echo "$RPMMACROS" >> $CHDIR/$CHHOME/.rpmmacros
 343
 344         cp -a $CHDIR/$CHHOME/{.bashrc,.builderrc}
 345         cat <<-'EORC' > $CHDIR/$CHHOME/.builderrc
 346         TITLECHANGE=no
 347 EORC
 348         [ -z "$BUILDERRC" ] || echo "$BUILDERRC" >> $CHDIR/$CHHOME/.builderrc
 349
 350         set +e
 351 }
 352
 353 build_mount_home()
 354 {
 355         $NODEBUG || set -x
 356         mount -o bind $HOME/rpm $CHDIR/$CHHOME/rpm
 357
 358         # ensure RPMS dir is available
 359         chroot $CHDIR su $USER -c 'mkdir -p $(rpm -E %_rpmdir)'
 360 }
 361
 362 print_installed()
 363 {
 364         echo=$1
 365         if [ -r installed/$build_pkg ]; then
 366                 $echo "$(cat installed/$build_pkg | awk '{print $1}' | sort -u \
 367                         | awk '{br=br ", " $1} END{gsub(/^, /, "- BR: ", br ); print br}')"
 368                 cat installed/$build_pkg
 369         fi
 370 }
 371
 372 addlist()
 373 {
 374         LIST="$1"
 375
 376         print_installed info
 377
 378         return
 379         echo "*** $build_pkg $(date --rfc-3339=seconds) ***" >> $LIST
 380         print_installed echo >> $LIST
 381 }
 382
 383 builddie()
 384 {
 385         LIST="$1"; shift
 386         CODE="$1"; shift
 387         MSG="$*"
 388
 389         rm -f $CHDIR/.pid
 390
 391         $CLEANAFTER && clean
 392         title "failed !"
 393
 394         addlist "ERROR_$LIST"
 395         die $CODE "$MSG"
 396 }
 397
 398 LAST_INSTALL=""
 399 poldek_install()
 400 {
 401         local I="$1" ret
 402         # Nothing to install
 403         [ -n "$I" ] || return 1
 404         # Installing same packets second time
 405         [ "$LAST_INSTALL" != "$I" ] || return 1
 406         LAST_INSTALL="$I"
 407
 408         info "Installing" $I
 409         poldek -O "ignore=$IGNORE" -u $I | tee $$.poldek_install
 410         ret=
 411         if grep -q "Preparing...                ##################################################" $$.poldek_install \
 412                         && ! grep -q "file .* from install of .* conflicts with file from package" $$.poldek_install
 413                 then
 414                 info "Poldek:" $I "installed"
 415                 ret=0
 416         elif grep -q "Nothing to do" $$.poldek_install; then
 417                 warn "Poldek:" $I "installed already"
 418                 ret=1
 419         fi
 420         rm $$.poldek_install
 421         [ -n "$ret" ] && return $ret
 422
 423         # try harder
 424         info "Poldek install failed, retry without ignore"
 425         poldek -u $I && return 0
 426         info "Poldek install failed, retry once more without ignore"
 427         poldek -u $I && return 0
 428         warn "Poldek:" "Could not install" $I
 429         return 1
 430 }
 431
 432 maybe_call()
 433 {
 434         local cond="$1"; shift
 435         local func="$1"; shift
 436
 437         [ $cond = "false" ] && return
 438         "$func" "$@"
 439         [ $cond = "exit_after" ] && exit
 440 }
 441
 442 fetch()
 443 {
 444         info "Fetching $build_pkg"
 445         title "fetch"
 446         $NODEBUG || set -x
 447         su $USER -c "$HOME/rpm/packages/builder -g $build_pkg $builder_options" \
 448                 || die 11 "Fetch failed"
 449 }
 450
 451 create()
 452 {
 453         $NODEBUG || set -x
 454         su $USER -c "poldek -s $RPMS_FROM --mkidx"
 455
 456         if [ ! -d $CHDIR ]; then
 457                 info "Preparing $CHNAME"
 458                 build_prepare_root
 459                 build_mount_home
 460         fi
 461 }
 462
 463
 464 info "Configured Poldek sources"
 465 poldek -l
 466
 467 maybe_call $FETCH fetch
 468
 469 check_running
 470
 471 maybe_call $CLEAN clean
 472
 473 maybe_call $CREATE create
 474
 475 echo $$ > $CHDIR/.pid
 476
 477 maybe_call $INSTALL poldek_install "$*"
 478
 479 maybe_call $NETWORK cp -bf /etc/resolv.conf $CHDIR/etc/
 480
 481 $BUILD || exit
 482
 483 if [ -p /tmp/fixfreq ]; then
 484         echo $$ > /tmp/fixfreq
 485 fi
 486
 487 while true; do
 488         info "Building $build_pkg in $CHNAME"
 489         rebuilddb
 490         find $CHDIR/usr/lib{,64} -name "*.la" -print0 | \
 491                 xargs -0 -r sed -i -e "s@dependency_libs=.*@dependency_libs=' '@"
 492         buildlog="buildlogs/$build_pkg"
 493         if [ -r $buildlog ]; then
 494                 i=1
 495                 while [ -r $buildlog.$i ]; do
 496                         i=$((i+1))
 497                 done
 498                 info "moving $buildlog to $buildlog.$i"
 499                 mv $buildlog $buildlog.$i
 500         fi
 501         ./findunusedbr -c $CHDIR $HOME/rpm/packages/$build_pkg/$build_pkg.spec
 502         title "building"
 503         ./teeboth $buildlog chroot $CHDIR su $USER -c "$CHHOME/rpm/packages/builder -nn --define '_enable_debug_packages 0' -bb $build_pkg $builder_options"
 504         ECODE=$?
 505
 506         if grep -q "error: Failed build dependencies:" $buildlog; then
 507                 SEARCH=$(cat $buildlog | awk '/^Error:/ { p = 0 }; { if ( p ) { f="p"; if ( $1 ~ /^\// ) f="f"; printf "search -%c %s; ", f, $1; } }; /error: Failed build dependencies:/ { p = 1 }')
 508                 INSTALL=$(poldek -O "ignore=$IGNORE" --shcmd="$SEARCH" | awk '{ if ( p ) { print; p = 0; } } / package\(s\) found:$/ { p = 1 }' | sed 's/^\(.*\)-.*-.*$/\1/' | sort -u)
 509
 510                 if poldek_install "$INSTALL"; then
 511                         info "Deps installed"
 512                         continue
 513                 else
 514                         addlist ERROR_BRINSTALL
 515                         die 4 "Cannot install BRs"
 516                 fi
 517         fi
 518
 519         ./findbr $CHDIR/$BUILDDIR $buildlog > $$.installed
 520         installed_something=false
 521         while read pkg msg; do
 522                 if poldek_install $pkg; then
 523                         info "findbr:" $pkg "installed"
 524                         echo "$pkg $msg" >> installed/$build_pkg
 525                         ./addbr $build_pkg "$pkg" "$msg"
 526                         installed_something=true
 527                 else
 528                         warn "findbr:" $pkg "not installed"
 529                 fi
 530         done < $$.installed
 531         rm -f $$.installed
 532         $installed_something && continue
 533
 534         if [ $ECODE -eq 0 ]; then
 535                 $CLEANAFTER && clean
 536                 addlist BUILT_OK
 537                 ./findunusedbr $CHDIR $HOME/rpm/packages/$build_pkg/$build_pkg.spec
 538                 info "$build_pkg built OK !"
 539                 title "OK !"
 540                 exit 0
 541         else
 542                 builddie UNKNOWN 1 "Got error but dunno what to do !"
 543         fi
 544 done
 545
 546
 547 # vim: ts=4 sw=4 filetype=sh