catalyst-auto: move all code into functions

Mixing inline funcs and executable code makes it hard to follow and
shuffle ordering of operations.  Put everything other than variables
into functions.

The delta here is large, but it's almost entirely indentation changes.

This also makes updating the script (like `git pull`) more resilient.
This commit is contained in:
Mike Frysinger 2018-01-15 20:11:44 -05:00
parent b5e4453a0e
commit b291ea4754

View file

@ -19,6 +19,17 @@ unset UNSHARE
CATALYST_CONFIG=/etc/catalyst/catalyst.conf CATALYST_CONFIG=/etc/catalyst/catalyst.conf
# Probe the default source dir from this script name.
REPO_DIR=$(dirname "$(dirname "$(realpath "$0")")")
# Set up defaults that config files can override if they want.
SUBARCH=$(uname -m)
EMAIL_TO="releng@gentoo.org,gentoo-releng-autobuilds@lists.gentoo.org"
# Use full hostname by default as Gentoo servers will reject short names.
EMAIL_FROM="catalyst@$(hostname -f)"
EMAIL_SUBJECT_PREPEND="[${SUBARCH}-auto]"
# Variables updated by command line arguments.
declare -a config_files declare -a config_files
config_files=() config_files=()
verbose=0 verbose=0
@ -28,9 +39,6 @@ preclean=0
lastrun=0 lastrun=0
lock_file= lock_file=
# Set pipefail so that run_cmd returns the right value in $?
set -o pipefail
usage() { usage() {
local msg=$1 local msg=$1
@ -125,269 +133,271 @@ trigger_post_build() {
fi fi
} }
# Parse args parse_args() {
while [ ${#} -gt 0 ] local a
do while [[ $# -gt 0 ]] ; do
a=${1} a=$1
shift shift
case "${a}" in case "${a}" in
-h|--help) -h|--help)
usage usage
exit 0 exit 0
;; ;;
-c|--config) -c|--config)
config_files+=("$1") config_files+=("$1")
shift shift
;; ;;
-v|--verbose) -v|--verbose)
verbose=$(($verbose+1)) verbose=$(($verbose+1))
;; ;;
-k|--keep-tmpdir) -k|--keep-tmpdir)
keep_tmpdir=1 keep_tmpdir=1
;; ;;
-t|--test) -t|--test)
testing=1 testing=1
;; ;;
-C|--preclean) -C|--preclean)
preclean=1 preclean=1
;; ;;
--interval) --interval)
lastrun=$1 lastrun=$1
shift shift
;; ;;
-l|--lock) -l|--lock)
lock_file=$1 lock_file=$1
shift shift
;; ;;
-*) -*)
usage "ERROR: You have specified an invalid option: ${a}" usage "ERROR: You have specified an invalid option: ${a}"
exit 1 exit 1
;; ;;
*) *)
usage "ERROR: This script takes no arguments: ${a}" usage "ERROR: This script takes no arguments: ${a}"
exit 1 exit 1
;; ;;
esac esac
done done
}
( run_catalyst_commands() {
doneconfig=0
for config_file in "${config_files[@]}"; do
# Make sure all required values were specified.
if [[ -z "${config_file}" || ! -e "${config_file}" ]]; then
usage "ERROR: You must specify a valid config file to use: '$config_file' is not valid"
exit 1
fi
source "${config_file}"
doneconfig=1
done
if [[ ${doneconfig} == 0 ]]; then
usage "ERROR: You must specify at least one valid config file to use"
exit 1
fi
if [[ -n ${lock_file} ]]; then # Some configs will set this explicitly, so don't clobber it.
if ! flock -n 9; then : ${BUILD_SRCDIR_BASE:=$(catalyst_var storedir)}
echo "catalyst-auto already running"
exit 1
fi
fi
# Probe the default source dir from this script name. # See if we had a recent success.
REPO_DIR=$(dirname "$(dirname "$(realpath "$0")")") if [[ ${lastrun} != 0 ]]; then
last_success_file="${BUILD_SRCDIR_BASE}/.last_success"
delay=$(( lastrun * 24 * 60 * 60 ))
last_success=$(head -1 "${last_success_file}" 2>/dev/null || echo 0)
if [[ $(date +%s) -lt $(( last_success + delay )) ]]; then
exit 0
fi
fi
# Set up defaults that config files can override if they want. DATESTAMP=$(date -u +%Y%m%d)
SUBARCH=$(uname -m) TIMESTAMP=$(date -u +%Y%m%dT%H%M%SZ)
EMAIL_TO="releng@gentoo.org,gentoo-releng-autobuilds@lists.gentoo.org" TMPDIR=$(mktemp -d --tmpdir="${TMP_PATH:-/tmp}" "catalyst-auto.${TIMESTAMP}.XXXXXX")
# Use full hostname by default as Gentoo servers will reject short names.
EMAIL_FROM="catalyst@$(hostname -f)"
EMAIL_SUBJECT_PREPEND="[${SUBARCH}-auto]"
doneconfig=0 # Nuke any previous tmpdirs to keep them from accumulating.
for config_file in "${config_files[@]}"; do if [[ ${preclean} == 1 ]]; then
# Make sure all required values were specified rm -rf "${TMPDIR%.??????}".*
if [ -z "${config_file}" -o ! -e "${config_file}" ]; then mkdir "${TMPDIR}"
usage "ERROR: You must specify a valid config file to use: '$config_file' is not valid" fi
exit 1
fi
source "${config_file}"
doneconfig=1
done
if [[ $doneconfig -eq 0 ]]; then
usage "ERROR: You must specify at least one valid config file to use"
exit 1
fi
# Some configs will set this explicitly, so don't clobber it. if [[ ${verbose} -ge 1 ]]; then
: ${BUILD_SRCDIR_BASE:=$(catalyst_var storedir)} echo "TMPDIR = ${TMPDIR}"
echo "DATESTAMP = ${DATESTAMP}"
echo "TIMESTAMP = ${TIMESTAMP}"
fi
# See if we had a recent success. if ! mkdir -p "${TMPDIR}"/{specs,kconfig,log}; then
if [[ ${lastrun} -ne 0 ]]; then echo "Couldn't create tempdirs!"
last_success_file="${BUILD_SRCDIR_BASE}/.last_success" exit 1
delay=$(( lastrun * 24 * 60 * 60 )) fi
last_success=$(head -1 "${last_success_file}" 2>/dev/null || echo 0)
if [[ $(date +%s) -lt $(( last_success + delay )) ]]; then
exit 0
fi
fi
DATESTAMP=$(date -u +%Y%m%d) if ! run_cmd "${TMPDIR}/log/pre_build.log" pre_build; then
TIMESTAMP=$(date -u +%Y%m%dT%H%M%SZ) send_email "Catalyst build error - pre_build" "The pre_build function failed" "${TMPDIR}/log/pre_build.log"
TMPDIR=$(mktemp -d --tmpdir="${TMP_PATH:-/tmp}" "catalyst-auto.${TIMESTAMP}.XXXXXX") exit 1
fi
# Nuke any previous tmpdirs to keep them from accumulating. cd "${SPECS_DIR}" || exit 1
if [[ ${preclean} -eq 1 ]]; then
rm -rf "${TMPDIR%.??????}".*
mkdir "${TMPDIR}"
fi
if [ ${verbose} -ge 1 ]; then for a in "" ${SETS}; do
echo "TMPDIR = ${TMPDIR}" if [[ -z "${a}" ]]; then
echo "DATESTAMP = ${DATESTAMP}" specs_var="SPECS"
echo "TIMESTAMP = ${TIMESTAMP}" optional_specs_var="OPTIONAL_SPECS"
fi else
specs_var="SET_${a}_SPECS"
optional_specs_var="SET_${a}_OPTIONAL_SPECS"
fi
if ! mkdir -p "${TMPDIR}"/{specs,kconfig,log}; then for i in ${!specs_var} ${!optional_specs_var}; do
echo "Couldn't create tempdirs!" cp --parents "${i}" "${TMPDIR}"/specs/
exit 1 done
fi done
if ! run_cmd "${TMPDIR}/log/pre_build.log" pre_build; then find "${KCONFIG_DIR}" -type f -exec cp {} "${TMPDIR}"/kconfig \;
send_email "Catalyst build error - pre_build" "The pre_build function failed" "${TMPDIR}/log/pre_build.log"
exit 1
fi
cd "${SPECS_DIR}" || exit 1 cd "${TMPDIR}/specs" || exit 1
for a in "" ${SETS}; do # Fix up specs with datestamp
if [ -z "${a}" ]; then for i in $(find -name '*.spec'); do
specs_var="SPECS" # Grab current version_stamp and source_subpath
optional_specs_var="OPTIONAL_SPECS" old_version_stamp=$(grep version_stamp "${i}" | sed -e 's|^version_stamp: *||')
else old_source_subpath=$(grep source_subpath "${i}" | sed -e 's|^source_subpath: *||')
specs_var="SET_${a}_SPECS"
optional_specs_var="SET_${a}_OPTIONAL_SPECS"
fi
for i in ${!specs_var} ${!optional_specs_var}; do new_version_stamp=$(echo "${old_version_stamp}" | sed -e "s|^\(.*-\)\?.*$|\1${TIMESTAMP}|")
cp --parents "${i}" "${TMPDIR}"/specs/ new_source_subpath=$(echo "${old_source_subpath}" | sed -e "s|${old_version_stamp}|${new_version_stamp}|")
done
done
find "${KCONFIG_DIR}" -type f -exec cp {} "${TMPDIR}"/kconfig \; sed -i "s|^version_stamp:.*$|version_stamp: ${new_version_stamp}|" "${i}"
sed -i "s|^snapshot:.*$|snapshot: ${TIMESTAMP}|" "${i}"
cd "${TMPDIR}/specs" || exit 1 # We don't want to mangle the source_subpath for our stage1 spec
if ! grep -q '^target: *stage[14]$' "${i}"; then
sed -i "s|^source_subpath:.*$|source_subpath: ${new_source_subpath}|" "${i}"
fi
# Fix up specs with datestamp sed -i "/^livecd\/iso/s|${old_version_stamp}|${new_version_stamp}|" "${i}"
for i in $(find -name '*.spec'); do sed -i "/^livecd\/volid/s|${old_version_stamp}|${new_version_stamp}|" "${i}"
# Grab current version_stamp and source_subpath
old_version_stamp=$(grep version_stamp "${i}" | sed -e 's|^version_stamp: *||')
old_source_subpath=$(grep source_subpath "${i}" | sed -e 's|^source_subpath: *||')
new_version_stamp=$(echo "${old_version_stamp}" | sed -e "s|^\(.*-\)\?.*$|\1${TIMESTAMP}|") kconfig_lines=$(grep '^boot/kernel/[^/]\+/config:' "${i}")
new_source_subpath=$(echo "${old_source_subpath}" | sed -e "s|${old_version_stamp}|${new_version_stamp}|") if [[ -n ${kconfig_lines} ]]; then
echo "${kconfig_lines}" | while read line; do
key=$(echo "${line}" | cut -d: -f1)
filename=$(basename $(echo "${line}" | cut -d: -f2))
sed -i "s|^${key}:.*\$|${key}: ${TMPDIR}/kconfig/${filename}|" "${i}"
done
fi
sed -i "s|^version_stamp:.*$|version_stamp: ${new_version_stamp}|" "${i}" # Expand vars that the spec expects us to.
sed -i "s|^snapshot:.*$|snapshot: ${TIMESTAMP}|" "${i}" sed -i \
-e "s:@DATESTAMP@:${DATESTAMP}:g" \
-e "s:@TIMESTAMP@:${TIMESTAMP}:g" \
-e "s:@REPO_DIR@:${REPO_DIR}:g" \
"${i}"
done
# We don't want to mangle the source_subpath for our stage1 spec if [[ ${testing} == 1 ]]; then
if ! grep -q '^target: *stage[14]$' "${i}"; then echo "Exiting due to --test"
sed -i "s|^source_subpath:.*$|source_subpath: ${new_source_subpath}|" "${i}" exit
fi fi
sed -i "/^livecd\/iso/s|${old_version_stamp}|${new_version_stamp}|" "${i}" if [[ ${preclean} == 1 ]]; then
sed -i "/^livecd\/volid/s|${old_version_stamp}|${new_version_stamp}|" "${i}" snapshot_cache=$(catalyst_var snapshot_cache)
if [[ -z ${snapshot_cache} ]]; then
echo "error: snapshot_cache not set in config file"
exit 1
fi
pushd "${BUILD_SRCDIR_BASE}" >/dev/null || exit 1
rm -rf --one-file-system \
kerncache packages snapshots tmp "${snapshot_cache}"/*
popd >/dev/null
fi
kconfig_lines=$(grep '^boot/kernel/[^/]\+/config:' "${i}") # Create snapshot
if [ -n "${kconfig_lines}" ]; then if ! run_cmd "${TMPDIR}/log/snapshot.log" catalyst -c "${CATALYST_CONFIG}" -s "${TIMESTAMP}"; then
echo "${kconfig_lines}" | while read line; do send_email "Catalyst build error - snapshot" "" "${TMPDIR}/log/snapshot.log"
key=$(echo "${line}" | cut -d: -f1) exit 1
filename=$(basename $(echo "${line}" | cut -d: -f2)) fi
sed -i "s|^${key}:.*\$|${key}: ${TMPDIR}/kconfig/${filename}|" "${i}"
done
fi
# Expand vars that the spec expects us to. build_failure=0
sed -i \
-e "s:@DATESTAMP@:${DATESTAMP}:g" \
-e "s:@TIMESTAMP@:${TIMESTAMP}:g" \
-e "s:@REPO_DIR@:${REPO_DIR}:g" \
"${i}"
done
if [ "${testing}" -eq 1 ]; then timeprefix=()
echo "Exiting due to --test" which time >/dev/null && timeprefix=( "time" )
exit
fi
if [[ ${preclean} -eq 1 ]]; then for a in "" ${SETS}; do
snapshot_cache=$(catalyst_var snapshot_cache) if [[ -z ${a} ]]; then
if [[ -z ${snapshot_cache} ]]; then specs_var="SPECS"
echo "error: snapshot_cache not set in config file" optional_specs_var="OPTIONAL_SPECS"
exit 1 else
fi specs_var="SET_${a}_SPECS"
pushd "${BUILD_SRCDIR_BASE}" >/dev/null || exit 1 optional_specs_var="SET_${a}_OPTIONAL_SPECS"
rm -rf --one-file-system \ fi
kerncache packages snapshots tmp "${snapshot_cache}"/*
popd >/dev/null
fi
# Create snapshot for i in ${!specs_var}; do
if ! run_cmd "${TMPDIR}/log/snapshot.log" catalyst -c "${CATALYST_CONFIG}" -s "${TIMESTAMP}"; then LOGFILE="${TMPDIR}/log/$(echo "${i}" | sed -e 's:/:_:' -e 's:\.spec$::').log"
send_email "Catalyst build error - snapshot" "" "${TMPDIR}/log/snapshot.log" run_cmd "${LOGFILE}" "${timeprefix[@]}" catalyst -a -c "${CATALYST_CONFIG}" -f "${i}"
exit 1 if [[ $? != 0 ]]; then
fi build_failure=1
send_email "Catalyst fatal build error - ${i}" "" "${LOGFILE}"
continue 2
else
trigger_post_build
fi
done
build_failure=0 for i in ${!optional_specs_var}; do
LOGFILE="${TMPDIR}/log/$(echo "${i}" | sed -e 's:/:_:' -e 's:\.spec$::').log"
run_cmd "${LOGFILE}" "${timeprefix[@]}" catalyst -a -c "${CATALYST_CONFIG}" -f "${i}"
if [[ $? != 0 ]]; then
build_failure=1
send_email "Catalyst non-fatal build error - ${i}" "" "${LOGFILE}"
break
else
trigger_post_build
fi
done
timeprefix=() for i in ${!specs_var} ${!optional_specs_var}; do
which time >/dev/null && timeprefix=( "time" ) LOGFILE="${TMPDIR}/log/$(echo "${i}" | sed -e 's:/:_:' -e 's:\.spec$::')_purge.log"
run_cmd "${LOGFILE}" "${timeprefix[@]}" catalyst --purgetmponly -c "${CATALYST_CONFIG}" -f "${i}"
done
for a in "" ${SETS}; do update_symlinks
if [ -z "${a}" ]; then done
specs_var="SPECS"
optional_specs_var="OPTIONAL_SPECS"
else
specs_var="SET_${a}_SPECS"
optional_specs_var="SET_${a}_OPTIONAL_SPECS"
fi
for i in ${!specs_var}; do trigger_post_build
LOGFILE="${TMPDIR}/log/$(echo "${i}" | sed -e 's:/:_:' -e 's:\.spec$::').log"
run_cmd "${LOGFILE}" "${timeprefix[@]}" catalyst -a -c "${CATALYST_CONFIG}" -f "${i}"
if [ $? != 0 ]; then
build_failure=1
send_email "Catalyst fatal build error - ${i}" "" "${LOGFILE}"
continue 2
else
trigger_post_build
fi
done
for i in ${!optional_specs_var}; do if [[ ${build_failure} == 0 ]]; then
LOGFILE="${TMPDIR}/log/$(echo "${i}" | sed -e 's:/:_:' -e 's:\.spec$::').log" if [[ ${lastrun} != 0 ]]; then
run_cmd "${LOGFILE}" "${timeprefix[@]}" catalyst -a -c "${CATALYST_CONFIG}" -f "${i}" stamp=$(date)
if [ $? != 0 ]; then (date -d"${stamp}" +%s; echo "${stamp}") >"${last_success_file}"
build_failure=1 fi
send_email "Catalyst non-fatal build error - ${i}" "" "${LOGFILE}"
break
else
trigger_post_build
fi
done
for i in ${!specs_var} ${!optional_specs_var}; do send_email "Catalyst build success" "Build process complete."
LOGFILE="${TMPDIR}/log/$(echo "${i}" | sed -e 's:/:_:' -e 's:\.spec$::')_purge.log"
run_cmd "${LOGFILE}" "${timeprefix[@]}" catalyst --purgetmponly -c "${CATALYST_CONFIG}" -f "${i}"
done
update_symlinks if [[ ${keep_tmpdir} == 0 ]]; then
done if ! rm -rf "${TMPDIR}"; then
echo "Could not remove tmpdir ${TMPDIR}!"
exit 1
fi
fi
else
send_email "Catalyst build complete, but with errors" "Build process has completed, but there were errors. Please consult previous emails to determine the problem."
fi
}
trigger_post_build main() {
# Set pipefail so that run_cmd returns the right value in $?.
set -o pipefail
if [ ${build_failure} = 0 ]; then # Parse user arguments before we try doing container logic.
if [[ ${lastrun} -ne 0 ]]; then parse_args "$@"
stamp=$(date)
(date -d"${stamp}" +%s; echo "${stamp}") >"${last_success_file}"
fi
send_email "Catalyst build success" "Build process complete." (
if [[ -n ${lock_file} ]]; then
if ! flock -n 9; then
echo "catalyst-auto already running"
exit 1
fi
fi
if [ "${keep_tmpdir}" = 0 ]; then run_catalyst_commands
if ! rm -rf "${TMPDIR}"; then ) 9>"${lock_file:-/dev/null}"
echo "Could not remove tmpdir ${TMPDIR}!" }
exit 1
fi
fi
else main "$@"
send_email "Catalyst build complete, but with errors" "Build process has completed, but there were errors. Please consult previous emails to determine the problem."
fi
) 9>"${lock_file:-/dev/null}"