gentoo-asahi-releng/tools/catalyst-auto

#!/bin/bash
# Copyright 1999-2014 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2

CATALYST_CONFIG=/etc/catalyst/catalyst.conf

# Probe the default source dir from this script name.
REPO_DIR=$(dirname "$(dirname "$(realpath "$0")")")

# Set up defaults that config files can override if they want.
SUBARCH=$(uname -m)
EMAIL_TO="releng@gentoo.org,gentoo-releng-autobuilds@lists.gentoo.org"
# Use full hostname by default as Gentoo servers will reject short names.
EMAIL_FROM="catalyst@$(hostname -f)"
EMAIL_SUBJECT_PREPEND="[${SUBARCH}-auto]"

# Variables updated by command line arguments.
declare -a config_files
config_files=()
verbose=0
keep_tmpdir=0
testing=0
preclean=0
lastrun=0
lock_file=
parallel_sets=1
nonetwork=0

usage() {
	local msg=$1

	if [ -n "${msg}" ]; then
		printf "%b\n\n" "${msg}"
	fi

	cat <<EOH
Usage:
  catalyst-auto [-c|--config <config>] [-v|--verbose] [-h|--help]

Options:
  -c|--config         Specifies the config file to use (required)
  -C|--preclean       Clean up loose artifacts from previous runs
  -j|--jobs <n>       Build <n> spec sets in parallel
  -v|--verbose        Send output of commands to console as well as log
  -k|--keep-tmpdir    Don't remove temp dir when build finishes
  -t|--test           Stop after mangling specs and copying files
  --interval <days>   Exit if last successful run was less than <days> ago
  -l|--lock <file>    File to grab a lock on to prevent multiple invocations
  -X|--nonetwork      Do not perform network operations (like uploading result)
  -h|--help           Show this message and quit

EOH
}

send_email() {
	if [[ ${verbose} -ge 1 ]]; then
		echo "$1 $2 build log at $3"
	fi

	[[ ${nonetwork} == 0 ]] || return

	[[ ! -z ${EMAIL_TO} ]] || return

	local subject="${EMAIL_SUBJECT_PREPEND} $1"
	local message=$2
	local logfile=$3
	local body

	if [ -n "${logfile}" ]; then
		body=$(printf '%b\n\n\n' "${message}"; tail -n 1000 "${logfile}"; printf '\n\n\nFull build log at %s\n' "${logfile}")
	else
		body=${message}
	fi

	printf 'From: %s\r\nTo: %s\r\nSubject: %s\r\n\r\n%b' \
		"${EMAIL_FROM}" "${EMAIL_TO}" "${subject}" "${body}" | \
		/usr/sbin/sendmail -f "${EMAIL_FROM}" ${EMAIL_TO//,/ }
}

# Usage: run_cmd <logfile> <command to run>
run_cmd() {
	local logfile="$1"
	shift

	echo "*** Running command: $*" &>> "${logfile}"
	if [[ ${verbose} == 2 ]]; then
		echo "*** Running command: $*"
		"$@" 2>&1 | tee -a "${logfile}"
	elif [[ ${verbose} == 1 ]]; then
		echo "*** Running command: $*"
		"$@" &>> "${logfile}"
	else
		"$@" &>> "${logfile}"
	fi

	# If we used tee above, make sure we pass back up the command's error.
	return ${PIPESTATUS[0]}
}

update_symlinks() {
	# This is a skeleton function that you can override from the config file.
	# It will be called by pre_build and after completing the build of a set
	# to ensure the symlinks point to the latest built stages.
	:
}

pre_build() {
	# This is a skeleton function that you can override from the config file.
	# It will be executed before the build is started.
	update_symlinks
}

post_build() {
	# This is a skeleton function that you can override from the config file.
	# It will be executed after the build is successfully completed. You can
	# use this to rsync the builds to another box
	:
}

catalyst_var() {
	# Extract a setting from the catalyst.conf.
	local var=$(grep --color=never -Po "^${1}\s*=\s*\K.*" "${CATALYST_CONFIG}" || true)
	local vas=$(echo $var|sed -e 's:^"::g' -e 's:"$::g')
	[[ -z ${vas} ]] || echo "${vas}"
}

trigger_post_build() {
	local set=$1 spec=$2
	if ! run_cmd "${TMPDIR}/log/post_build.log" post_build "${set}" "${spec}"; then
		send_email "Catalyst build error - post_build" "The post_build function failed" "${TMPDIR}/log/post_build.log"
		exit 1
	fi
}

parse_args() {
	local a
	while [[ $# -gt 0 ]] ; do
		a=$1
		shift
		case "${a}" in
		-h|--help)
			usage
			exit 0
			;;
		-c|--config)
			config_files+=("$1")
			shift
			;;
		-j|--jobs)
			parallel_sets="$1"
			shift
			;;
		-v|--verbose)
			verbose=$(($verbose+1))
			;;
		-k|--keep-tmpdir)
			keep_tmpdir=1
			;;
		-t|--test)
			testing=1
			;;
		-C|--preclean)
			preclean=1
			;;
		-X|--nonetwork)
			nonetwork=1
			;;
		--interval)
			lastrun=$1
			shift
			;;
		-l|--lock)
			lock_file=$1
			shift
			;;
		-*)
			usage "ERROR: You have specified an invalid option: ${a}"
			exit 1
			;;
		*)
			usage "ERROR: This script takes no arguments: ${a}"
			exit 1
			;;
		esac
	done
}

give_latest_from_dates() {
	sed 's,-20,~20,g' | \
	sort -k +1 -n -t '~' | \
	awk -F'~' \
		'BEGIN{i=$1; o=$0};
		{ if($1 != i && i != "") { print o; }; i=$1; o=$0; }
		END { print o; };' | \
	tr '~' '-'
}

# Replace the date/time stamp in the filename to "latest".
# Forms we handle:
# stage3-xxx-2018.0.tar.bz2
# stage3-xxx-20180116.tar.bz2
# stage3-xxx-20180116T015819Z.tar.bz2
convert_filename() {
	sed -E 's:-20[0-9]+(\.[0-9]+|T[0-9]+Z)?:-latest:g'
}

# Let's get our own namespaces/etc... to avoid leaking crap.
containerize() {
	# If we've already relaunched, nothing to do.
	if [[ ${UNSHARE} == "true" ]] ; then
		return
	fi

	# Most systems have unshare available, but just in case.
	if type -P unshare >&/dev/null ; then
		local uargs=()
		# Probe the namespaces as some can be disabled (or we are not root).
		unshare -m -- true >&/dev/null && uargs+=( -m )
		unshare -u -- true >&/dev/null && uargs+=( -u )
		unshare -i -- true >&/dev/null && uargs+=( -i )
		unshare -p -- true >&/dev/null && uargs+=( -p -f --mount-proc )
		# Re-exec ourselves in the new namespace.
		UNSHARE=true exec unshare "${uargs[@]}" -- "$0" "$@"
	fi
}

# Update the git repo if possible.  It might modify this script which will probably
# make bash fail (since bash parses as it executes).  So we have to safely re-exec
# the script whenever there's an update.
git_update() {
	# If we've already relaunched, nothing to do.
	if [[ ${GIT_UPDATE} == "true" || ${nonetwork} == 1 ]] ; then
		return
	fi

	pushd "${REPO_DIR}" >/dev/null
	git fetch -q
	revs=$(git rev-list HEAD..FETCH_HEAD)
	popd >/dev/null
	if [[ -n ${revs} ]] ; then
		GIT_UPDATE=true exec bash -c '
			repo_dir=$1 script=$2
			shift 2
			pushd "${repo_dir}" >/dev/null
			git merge -q FETCH_HEAD || echo "${script}: WARNING: git repo is dirty"
			popd >/dev/null
			exec "${script}" "$@"
		' -- "${REPO_DIR}" "$0" "$@"
	fi
}

# Stages are uploaded to <arch>@releng-incoming.gentoo.org and in order to
# allow us to change what system this domain points to, we will retrieve the
# SSH fingerprint from DNS. To do this securely, we need to ensure DNSSEC is
# working.
verify_dnssec() {
	[[ ${nonetwork} == 0 ]] || return

	which dig >/dev/null || {
		echo "net-dns/bind-tools is needed to verify DNSSEC is working"
		exit 1
	}

	if ! dig +noall +comments dev.gentoo.org. IN SSHFP | egrep -q '^;; flags: [ a-z]+\<ad\>'; then
		echo "DNSSEC does not appear to be working. Bailing out"
		exit 1
	fi

	if ! grep -q '^options\>.*\<edns0\>' /etc/resolv.conf; then
		echo "DNSSEC is not enabled in /etc/resolv.conf"
		exit 1
	fi
}

upload() {
	if [[ ${nonetwork} == 0 ]]; then
		echo Uploading "$@"
		local SSH_CMD=(
			ssh
			-i ${UPLOAD_KEY}
			-o UserKnownHostsFile=/dev/null
			-o VerifyHostKeyDNS=yes
			-o StrictHostKeyChecking=no
			-o IPQoS=cs0
		)
		local RSYNC_OPTS=(
			-e "${SSH_CMD[*]}"
			--archive
			--omit-dir-times
			--delay-updates
		)
		rsync "${RSYNC_OPTS[@]}" "$@" ${UPLOAD_USER}@releng-incoming.gentoo.org:${UPLOAD_DEST}
	else
		echo Would now upload "$@"
		ls -l $@
	fi
}

upsync_binpackages() {
	# parameter 1: a PKGDIR on the local host
	# parameter 2: the target dir in the mirroring system, should be of the
	#              form arch/profileversion/name (e.g., amd64/17.0/x32 )
	if [[ ${nonetwork} == 0 ]]; then
		echo Upsyncing binpackages from $1 to $2
		local SSH_CMD=(
			ssh
			-i ${UPLOAD_KEY}
			-o UserKnownHostsFile=/dev/null
			-o VerifyHostKeyDNS=yes
			-o StrictHostKeyChecking=no
			-o IPQoS=cs0
		)
		local RSYNC_OPTS=(
			-e "${SSH_CMD[*]}"
			--archive
			--delete
			--delete-after
			--omit-dir-times
			--delay-updates
		)
		rsync "${RSYNC_OPTS[@]}" "$1" "${UPLOAD_USER}@releng-incoming.gentoo.org:/release/weekly/binpackages/$2"
	else
		echo Would now upsync binpackages from $1 to $2
		ls -l $@
	fi
}

run_catalyst_commands() {
	doneconfig=0
	for config_file in "${config_files[@]}"; do
		# Make sure all required values were specified.
		if [[ -z "${config_file}" || ! -e "${config_file}" ]]; then
			usage "ERROR: You must specify a valid config file to use: '$config_file' is not valid"
			exit 1
		fi
		source "${config_file}"
		doneconfig=1
	done
	if [[ ${doneconfig} == 0 ]]; then
		usage "ERROR: You must specify at least one valid config file to use"
		exit 1
	fi

	# Some configs will set this explicitly, so don't clobber it.
	: ${BUILD_SRCDIR_BASE:=$(catalyst_var storedir)}
	: ${BUILD_SRCDIR_BASE:=/var/tmp/catalyst}

	# See if we had a recent success.
	if [[ ${lastrun} != 0 ]]; then
		last_success_file="${BUILD_SRCDIR_BASE}/.last_success"
		delay=$(( lastrun * 24 * 60 * 60 ))
		last_success=$(head -1 "${last_success_file}" 2>/dev/null || echo 0)
		if [[ $(date +%s) -lt $(( last_success + delay )) ]]; then
			exit 0
		fi
	fi

	# Nuke any previous tmpdirs to keep them from accumulating.
	if [[ ${preclean} == 1 ]]; then
		rm -rf "${TMP_PATH:-/tmp}/catalyst-auto".*

		pushd "${BUILD_SRCDIR_BASE}" >/dev/null || exit 1
		rm -rf --one-file-system \
			kerncache packages snapshots tmp
		popd >/dev/null
	fi

	if catalyst --help | grep -q "git-treeish"; then
		snapshot_log=$(mktemp --tmpdir="${TMP_PATH:-/tmp}")
		if ! run_cmd "${snapshot_log}" catalyst -c "${CATALYST_CONFIG}" -s stable; then
			send_email "Catalyst build error - snapshot" "" "${snapshot_log}"
			exit 1
		fi

		read TREEISH gitdir <<<$(egrep -o 'Creating .* tree snapshot [0-9a-f]{40} from .*' "${snapshot_log}" | cut -d' ' -f5,7)
		TIMESTAMP=$(git -C "${gitdir}" show --no-patch --format=%cd --date=format:%Y%m%dT%H%M%SZ "${TREEISH}")
	else
		TIMESTAMP=$(date -u +%Y%m%dT%H%M%SZ)

		snapshot_log=$(mktemp --tmpdir="${TMP_PATH:-/tmp}")
		if ! run_cmd "${snapshot_log}" catalyst -c "${CATALYST_CONFIG}" -s "${TIMESTAMP}"; then
			send_email "Catalyst build error - snapshot" "" "${snapshot_log}"
			exit 1
		fi
	fi
	DATESTAMP=$(date -u +%Y%m%d)
	TMPDIR=$(mktemp -d --tmpdir="${TMP_PATH:-/tmp}" "catalyst-auto.${TIMESTAMP}.XXXXXX")

	if [[ ${verbose} -ge 1 ]]; then
		echo "TMPDIR = ${TMPDIR}"
		echo "TIMESTAMP = ${TIMESTAMP}"
	fi

	if ! mkdir -p "${TMPDIR}"/{specs,kconfig,log}; then
		echo "Couldn't create tempdirs!"
		exit 1
	fi
	mv "${snapshot_log}" "${TMPDIR}/log/snapshot.log"

	if ! run_cmd "${TMPDIR}/log/pre_build.log" pre_build; then
		send_email "Catalyst build error - pre_build" "The pre_build function failed" "${TMPDIR}/log/pre_build.log"
		exit 1
	fi

	cd "${SPECS_DIR}" || exit 1

	for a in "" ${SETS}; do
		if [[ -z "${a}" ]]; then
			specs_var="SPECS"
			optional_specs_var="OPTIONAL_SPECS"
		else
			specs_var="SET_${a}_SPECS"
			optional_specs_var="SET_${a}_OPTIONAL_SPECS"
		fi

		for i in ${!specs_var} ${!optional_specs_var}; do
			cp --parents "${i}" "${TMPDIR}"/specs/
		done
	done

	[[ -n ${KCONFIG_DIR} ]] && find "${KCONFIG_DIR}" -type f -exec cp {} "${TMPDIR}"/kconfig \;

	cd "${TMPDIR}/specs" || exit 1

	# Fix up specs with datestamp
	for i in $(find -name '*.spec'); do
		kconfig_lines=$(grep '^boot/kernel/[^/]\+/config:' "${i}")
		if [[ -n ${kconfig_lines} ]]; then
			echo "${kconfig_lines}" | while read line; do
				key=$(echo "${line}" | cut -d: -f1)
				filename=$(basename $(echo "${line}" | cut -d: -f2))
				sed -i "s|^${key}:.*\$|${key}: ${TMPDIR}/kconfig/${filename}|" "${i}"
			done
		fi

		if [[ -n ${TREEISH} ]]; then
			sed -i -e "s|snapshot:.*|snapshot_treeish: ${TREEISH}|g" "${i}"
		fi

		# Expand vars that the spec expects us to.
		sed -i \
			-e "s:@TIMESTAMP@:${TIMESTAMP}:g" \
			-e "s:@REPO_DIR@:${REPO_DIR}:g" \
			-e "s:@TREEISH@:${TREEISH}:g" \
			"${i}"
	done

	if [[ ${testing} == 1 ]]; then
		echo "Exiting due to --test"
		exit
	fi

	build_failure=0

	# bash built-in time function is not used
	if ! which time >/dev/null ; then
		timeprefix=()
		echo "sys-process/time is optional for build resource utilization"
	else
		timeprefix=( "time" )
	fi

	JOB_PIDS=()
	JOB_RETS=()
	JOB_IDX_S=0
	JOB_IDX_E=0

	for a in "" ${SETS}; do

		if [[ $(( JOB_IDX_E - JOB_IDX_S )) == ${parallel_sets} ]] ; then
			wait ${JOB_PIDS[$(( JOB_IDX_S++ ))]}
			JOB_RETS+=( $? )
		fi

		(

			if [[ -z ${a} ]]; then
				specs_var="SPECS"
				optional_specs_var="OPTIONAL_SPECS"
			else
				specs_var="SET_${a}_SPECS"
				optional_specs_var="SET_${a}_OPTIONAL_SPECS"
			fi

			for i in ${!specs_var}; do
				LOGFILE="${TMPDIR}/log/$(echo "${i}" | sed -e 's:/:_:' -e 's:\.spec$::').log"
				specpath=$(readlink -f "${i}")
				run_cmd "${LOGFILE}" "${timeprefix[@]}" catalyst -a -c "${CATALYST_CONFIG}" -f "${specpath}"
				if [[ $? != 0 ]]; then
					build_failure=1
					send_email "Catalyst fatal build error - ${i}" "" "${LOGFILE}"
					exit 1
				else
					trigger_post_build "${a}" "${i}"
				fi
			done

			for i in ${!optional_specs_var}; do
				LOGFILE="${TMPDIR}/log/$(echo "${i}" | sed -e 's:/:_:' -e 's:\.spec$::').log"
				specpath=$(readlink -f "${i}")
				run_cmd "${LOGFILE}" "${timeprefix[@]}" catalyst -a -c "${CATALYST_CONFIG}" -f "${specpath}"
				if [[ $? != 0 ]]; then
					build_failure=1
					send_email "Catalyst non-fatal build error - ${i}" "" "${LOGFILE}"
					break
				else
					trigger_post_build "${a}" "${i}"
				fi
			done

			# Do not purge yet, because there might be interdendency between specs
			# in different build sets!

			update_symlinks

			exit ${build_failure}

		)&

		JOB_PIDS+=( $! )
		: $(( ++JOB_IDX_E ))


	done

	for (( i = JOB_IDX_S; i < JOB_IDX_E; ++i )) ; do
		wait ${JOB_PIDS[i]}
		JOB_RETS+=( $? )
	done
	build_failure=$(( 0 ${JOB_RETS[@]/#/+} ))


	# Now do the cleanup
	for a in "" ${SETS}; do
		if [[ -z ${a} ]]; then
			specs_var="SPECS"
			optional_specs_var="OPTIONAL_SPECS"
		else
			specs_var="SET_${a}_SPECS"
			optional_specs_var="SET_${a}_OPTIONAL_SPECS"
		fi
		for i in ${!specs_var} ${!optional_specs_var}; do
			LOGFILE="${TMPDIR}/log/$(echo "${i}" | sed -e 's:/:_:' -e 's:\.spec$::')_purge.log"
			specpath=$(readlink -f "${i}")
			run_cmd "${LOGFILE}" "${timeprefix[@]}" catalyst --purgetmponly -c "${CATALYST_CONFIG}" -f "${specpath}"
		done
		update_symlinks
	done

	trigger_post_build

	if [[ ${build_failure} == 0 ]]; then
		if [[ ${lastrun} != 0 ]]; then
			stamp=$(date)
			(date -d"${stamp}" +%s; echo "${stamp}") >"${last_success_file}"
		fi

		send_email "Catalyst build success" "Build process complete." "${TMPDIR}/log/post_build.log"

		if [[ ${keep_tmpdir} == 0 ]]; then
			if ! rm -rf "${TMPDIR}"; then
				echo "Could not remove tmpdir ${TMPDIR}!"
				exit 1
			fi
		fi
	fi
}

main() {
	# Set pipefail so that run_cmd returns the right value in $?.
	set -o pipefail

	# Parse user arguments before we try doing container logic.
	parse_args "$@"

	# Update the release git dir if possible.
	git_update "$@"

	# Verify DNSSEC works
	verify_dnssec

	# Try to isolate ourselves from the rest of the system.
	containerize "$@"

	(
	if [[ -n ${lock_file} ]]; then
		if ! flock -n 9; then
			echo "catalyst-auto already running"
			exit 1
		fi
	fi

	run_catalyst_commands
	) 9>"${lock_file:-/dev/null}"
}

main "$@"