add --network option

[projects/cleanbuild.git] / cleanbuild

#!/usr/bin/sudo /bin/sh

RPMS_FROM="$HOME/rpm/cleanRPMS.repo"
DEST="th"
SRC="-n th-x86_64-test"
SUFFIX=""
CACHEDIR="$PWD/poldekcache"
RPMMACROS=""
BUILDERRC=""
IGNORE=""
CHROOTSIZE="4G"
ignore() { IGNORE="$IGNORE $*"; }
NODEBUG=true
CLEANAFTER=false
FORCE_UMOUNT=false
NOREBUILDDB=true
MULTILIB=false
# Whatever you set here as value, consider that it may not be shorter than
# /usr/lib/debug (or /usr/src/debug) for debuginfo to work.
# You get "Only dest dir longer than base dir not supported" error otherwise.
BUILDDIR=/usr/src/BUILD

[ -r .cleanbuildrc ] && . ./.cleanbuildrc

[ -z "$USER" ] && echo "USER not defined" && exit 1
[ "$USER" = "root" ] && echo "USER must not be root" && exit 1

export LC_ALL=C
unset LANGUAGE
unset LANG

usage() {
	[ $# -gt 0 ] && echo "$*"
	echo "Usage:"
	echo "	./cleanbuild [cleanbuild options] specname [builder options]"
	echo "	./build [cleanbuild options] specname [builder options]"
	echo "	./clean [cleanbuild options]"
	echo "	./create [cleanbuild options]"
	echo "	./install [cleanbuild options] packages"
	echo ""
	echo "cleanbuild options:"
	echo " -32, -64, -th-i486  - select architecture"
	echo " --cleanafter | -ca  - clean after build"
	echo " --forceumount | -fu - force umount tmpfs"
	echo " --debug             - enable debug"
	echo " --network           - allow build to use networking"
	echo " -a, -b, -c, -d, -e  - select alternative chroot directory"
	exit 1
}

FETCH=false
CLEAN=false
CREATE=false
BUILD=false
NETWORK=false
INSTALL=false

case "$0" in
	*clean)
		CLEAN=exit_after
		;;
	*cleanbuild)
		FETCH=true
		CLEAN=true
		CREATE=true
		BUILD=true
		;;
	*build)
		CREATE=true
		BUILD=true
		;;
	*create)
		CLEAN=true
		CREATE=exit_after
		;;
	*install)
		CREATE=true
		INSTALL=exit_after
		;;
	*)
		usage
		;;
esac

while [ $# -gt 0 ]; do
	OPT="${1}"
	case "$OPT" in
		-32)	OPT="-th-i686" ;;
		-64)	OPT="-th-x86_64" ;;
		-th-32)	OPT="-th-i686" ;;
		-th-64)	OPT="-th-x86_64" ;;
		-ti-32)	OPT="-ti-i686" ;;
		-ti-64)	OPT="-ti-x86_64" ;;
		-ac)	OPT="-ac-amd64" ;;
		-ac-32)	OPT="-ac-i586" ;;
		-ac-64)	OPT="-ac-amd64" ;;
	esac

	V="${OPT#-}"
	case "$OPT" in
		-th-i[46]86 | -th-x86_64)
			DEST="$V"
			SRC="-n $V-ready"
			;;
		-th-i[46]86-test | -th-x86_64-test)
			DEST="$V"
			SRC="-n $V"
			;;
		-ti-i[56]86 | -ti-x86_64)
			DEST="$V"
			SRC="-n $V-ready"
			;;
		-ti-i[56]86-test | -ti-x86_64-test)
			DEST="$V"
			SRC="-n $V"
			;;
		-ac-amd64 | -ac-i[356]86 | -ac-athlon)
			DEST="$V"
			SRC="-n $V-ready"
			;;
		-[1-9]G | -[1-9][0-9]G )
			CHROOTSIZE="$V"
			;;
		--cleanafter | -ca)
			CLEANAFTER=true
			;;
		--debug)
			NODEBUG=false
			;;
		--network)
			NETWORK=true
			;;
		--forceumount | -fu)
			FORCE_UMOUNT=true
			;;
		-[a-e])
			SUFFIX="$OPT"
			;;
		-~*)
			SUFFIX="$V"
			;;
		*)
			break
			;;
	esac
	shift
done

if $BUILD; then
	[ $# -ne 0 ] || usage
	build_pkg="${1}"
	build_pkg="${build_pkg%/}"
	build_pkg="${build_pkg%.spec}"
	build_pkg="${build_pkg#*/}"
	shift

	builder_options="$*"
fi

$NODEBUG || set -x

CHNAME="chroot-$DEST$SUFFIX"
CHDIR="$PWD/$CHNAME"
CHHOME="/home/users/$USER"

warn()
{
	echo -n -e "\033[31;1m" >&2
	echo -n "$*" >&2
	echo -e "\033[0m" >&2
}

die()
{
	code=$1
	shift
	warn "$*"
	exit $code
}

info()
{
	echo -n -e "\033[32m"
	echo -n "$*"
	echo -e "\033[0m"
}

title()
{
	[ -t 1 ] || return 0
	local msg="$CHNAME: $build_pkg: $*"
	case "$TERM" in
		cygwin|xterm*)
			echo -ne "\033]1;$msg\007\033]2;$msg\007" >&2
			;;
		screen*)
			echo -ne "\033]0;$msg\007" >&2
			;;
	esac
	return 0
}

exit_after()
{
	return 0;
}

check_running()
{
	[ -r "$CHDIR/.pid" ] || return
	PID=$(< "$CHDIR/.pid")
	if [ -d /proc/$PID ]; then
		die 10 "Another process ($PID) already running in $CHNAME"
	fi
}

for D in installed buildlogs $CACHEDIR; do
	if [ ! -d "$D" ]; then
		info "mkdir $D"
		su $USER -c "mkdir -p $D" || die 13 "Cannot create work directories"
	fi
done

ignore \
	upstart\* \
	upstart \
	compat-\* \
	systemd-init \
	hhvm \
	vserver-packages \
	xorg-driver-video-fglrx\* xorg-driver-video-nvidia\* xorg-xserver-xgl-libGL \
	xorg-driver-video-vboxvideo \
	xorg-data-xbitmaps \
	compat-gcc\* \
	libpng1\* \
	libjpeg libjpeg-devel \
	freetype1-devel-* \
	anacron fcron hc-cron \
	masqmail msmtp-sendmail omta postfix sendmail ssmtp nail-mail nullmailer \
	ghostscript-esp \
	perl-Scalar-List-Utils \
	perl-ExtUtils-Install \
	phonon-backend-mplayer phonon-backend-vlc \
	icedtea6-jre icedtea6-jdk \
	java-sun-jre java5-sun-jre oracle-java7-jre \
	gnome-menus \
	gnome-speech-driver-festival gnome-speech-driver-speech-dispatcher

if ! $MULTILIB; then
	ignore '*-multilib-*'
fi

rebuilddb()
{
	$NOREBUILDDB || rpm --root=$CHDIR --rebuilddb
}

poldek()
{
	$NODEBUG || set -x
	rebuilddb
	/usr/bin/poldek $SRC -s "$RPMS_FROM" -r "$CHDIR" "--cachedir=$CACHEDIR" --conf=$PWD/poldekconf/poldek.conf "$@"
}


build_umount()
{
	for DIR in $CHHOME/rpm $CHHOME dev proc sys; do
		[ -d $CHDIR/$DIR ] && umount $CHDIR/$DIR
	done
}

build_remove_root()
{
	$NODEBUG || set -x
	if $FORCE_UMOUNT; then
		# safety checks.
		[ "$CHDIR" ] || exit 1
		[ -d "$CHDIR" ] || exit 1
		rm -rf $CHDIR/*
		umount -l $CHDIR
	else
		umount $CHDIR
	fi
	rmdir $CHDIR
}

clean()
{
	info "Cleaning $CHNAME"
	title "cleaning chroot"
	build_umount
	build_remove_root
}

build_prepare_root()
{
	title "preparing chroot"
	set -e
	$NODEBUG || set -x
	mkdir $CHDIR
	mount -t tmpfs -o size=$CHROOTSIZE,relatime /dev/null $CHDIR
	echo $$ > $CHDIR/.pid

	rpmversion=$(rpm -E '%(v=%{_rpmversion}; IFS=.; set -- $v; echo $1)')
	rpmversion=${rpmversion:-4}

	if [ "$rpmversion" -ge 5 ]; then
		rpm --root=$CHDIR -qa
	else
		rpm --root=$CHDIR --initdb
	fi
	poldek --up || :
	poldek -O "ignore=$IGNORE" -u rpm-build pwdutils coreutils time util-linux git-core gawk
	echo Poldek exit: $?

	for DIR in dev proc sys; do
		# We need to create these directories manually, because they are marked
		# as netsharedpath in cleanbuild poldek.conf
		mkdir $CHDIR/$DIR
		mount -o bind /$DIR $CHDIR/$DIR
	done

	# group 'users' may already exist, so ignore errors
	chroot $CHDIR groupadd $(id $USER -gn) -g$(id $USER -g) || :
	chroot $CHDIR useradd -m $USER -u$(id $USER -u) -g $(id $USER -gn)

	# replicate files which already belong to $USER
	# so they will have correct owner and permissions
	cp -a $CHDIR/$CHHOME/{tmp,rpm}
	cp -a $CHDIR/$CHHOME/tmp $CHDIR$BUILDDIR

	cp -a $CHDIR/$CHHOME/{.bashrc,.rpmmacros}
	cat <<-EOM > $CHDIR/$CHHOME/.rpmmacros
	%_builddir		$BUILDDIR
	%buildroot		%{_builddir}/%{name}-%{version}-root-%(id -u -n)
	%_rpmdirname	cleanRPMS
	%_rpmdir		%{expand:%%global _rpmdir %([ -d %{_topdir}/../%{_rpmdirname} ] && (cd %{_topdir}/../%{_rpmdirname}; pwd) || echo %{_topdir}/%{_rpmdirname})}%_rpmdir
	%distribution	CleanPLD
	%_binary_payload	w1.gzdio
EOM
	[ -z "$RPMMACROS" ] || echo "$RPMMACROS" >> $CHDIR/$CHHOME/.rpmmacros

	cp -a $CHDIR/$CHHOME/{.bashrc,.builderrc}
	cat <<-'EORC' > $CHDIR/$CHHOME/.builderrc
	TITLECHANGE=no
EORC
	[ -z "$BUILDERRC" ] || echo "$BUILDERRC" >> $CHDIR/$CHHOME/.builderrc

	set +e
}

build_mount_home()
{
	$NODEBUG || set -x
	mount -o bind $HOME/rpm $CHDIR/$CHHOME/rpm
}


print_installed()
{
	echo=$1
	if [ -r installed/$build_pkg ]; then
		$echo "$(cat installed/$build_pkg | awk '{print $1}' | sort -u \
			| awk '{br=br ", " $1} END{gsub(/^, /, "- BR: ", br ); print br}')"
		cat installed/$build_pkg
	fi
}

addlist()
{
	LIST="$1"

	print_installed info

	return
	echo "*** $build_pkg $(date --rfc-3339=seconds) ***" >> $LIST
	print_installed echo >> $LIST
}

builddie()
{
	LIST="$1"; shift
	CODE="$1"; shift
	MSG="$*"

	rm -f $CHDIR/.pid

	$CLEANAFTER && clean
	title "failed !"

	addlist "ERROR_$LIST"
	die $CODE "$MSG"
}

LAST_INSTALL=""
poldek_install()
{
	local I="$1" ret
	# Nothing to install
	[ -n "$I" ] || return 1
	# Installing same packets second time
	[ "$LAST_INSTALL" != "$I" ] || return 1
	LAST_INSTALL="$I"

	info "Installing" $I
	poldek -O "ignore=$IGNORE" -u $I | tee $$.poldek_install
	ret=
	if grep -q "Preparing...                ##################################################" $$.poldek_install \
			&& ! grep -q "file .* from install of .* conflicts with file from package" $$.poldek_install
		then
		info "Poldek:" $I "installed"
		ret=0
	elif grep -q "Nothing to do" $$.poldek_install; then
		warn "Poldek:" $I "installed already"
		ret=1
	fi
	rm $$.poldek_install
	[ -n "$ret" ] && return $ret

	# try harder
	info "Poldek install failed, retry without ignore"
	poldek -u $I && return 0
	info "Poldek install failed, retry once more without ignore"
	poldek -u $I && return 0
	warn "Poldek:" "Could not install" $I
	return 1
}

maybe_call()
{
	local cond="$1"; shift
	local func="$1"; shift

	[ $cond = "false" ] && return
	"$func" "$@"
	[ $cond = "exit_after" ] && exit
}

fetch()
{
	info "Fetching $build_pkg"
	title "fetch"
	$NODEBUG || set -x
	su $USER -c "$HOME/rpm/packages/builder -g $build_pkg $builder_options" \
		|| die 11 "Fetch failed"
}

create()
{
	$NODEBUG || set -x
	su $USER -c "poldek -s $RPMS_FROM --mkidx"

	if [ ! -d $CHDIR ]; then
		info "Preparing $CHNAME"
		build_prepare_root
		build_mount_home
	fi
}


info "Configured Poldek sources"
poldek -l

maybe_call $FETCH fetch

check_running

maybe_call $CLEAN clean

maybe_call $CREATE create

echo $$ > $CHDIR/.pid

maybe_call $INSTALL poldek_install "$*"

maybe_call $NETWORK cp -bf /etc/resolv.conf $CHDIR/etc/

$BUILD || exit

if [ -p /tmp/fixfreq ]; then
	echo $$ > /tmp/fixfreq
fi

while true; do
	info "Building $build_pkg in $CHNAME"
	rebuilddb
	find $CHDIR/usr/lib{,64} -name "*.la" -print0 | \
		xargs -0 -r sed -i -e "s@dependency_libs=.*@dependency_libs=' '@"
	buildlog="buildlogs/$build_pkg"
	if [ -r $buildlog ]; then
		i=1
		while [ -r $buildlog.$i ]; do
			i=$((i+1))
		done
		info "moving $buildlog to $buildlog.$i"
		mv $buildlog $buildlog.$i
	fi
	./findunusedbr -c $CHDIR $HOME/rpm/packages/$build_pkg/$build_pkg.spec
	title "building"
	./teeboth $buildlog chroot $CHDIR su $USER -c "$CHHOME/rpm/packages/builder -nn --define '_enable_debug_packages 0' -bb $build_pkg $builder_options"
	ECODE=$?

	if grep -q "error: Failed build dependencies:" $buildlog; then
		SEARCH=$(cat $buildlog | awk '/^Error:/ { p = 0 }; { if ( p ) { f="p"; if ( $1 ~ /^\// ) f="f"; printf "search -%c %s; ", f, $1; } }; /error: Failed build dependencies:/ { p = 1 }')
		INSTALL=$(poldek -O "ignore=$IGNORE" --shcmd="$SEARCH" | awk '{ if ( p ) { print; p = 0; } } / package\(s\) found:$/ { p = 1 }' | sed 's/^\(.*\)-.*-.*$/\1/' | sort -u)

		if poldek_install "$INSTALL"; then
			info "Deps installed"
			continue
		else
			addlist ERROR_BRINSTALL
			die 4 "Cannot install BRs"
		fi
	fi

	./findbr $CHDIR/$BUILDDIR $buildlog > $$.installed
	installed_something=false
	while read pkg msg; do
		if poldek_install $pkg; then
			info "findbr:" $pkg "installed"
			echo "$pkg $msg" >> installed/$build_pkg
			./addbr $build_pkg "$pkg" "$msg"
			installed_something=true
		else
			warn "findbr:" $pkg "not installed"
		fi
	done < $$.installed
	rm -f $$.installed
	$installed_something && continue

	if [ $ECODE -eq 0 ]; then
		$CLEANAFTER && clean
		addlist BUILT_OK
		./findunusedbr $CHDIR $HOME/rpm/packages/$build_pkg/$build_pkg.spec
		info "$build_pkg built OK !"
		title "OK !"
		exit 0
	else
		builddie UNKNOWN 1 "Got error but dunno what to do !"
	fi
done


# vim: ts=4 sw=4 filetype=sh