f98bc791e6
Commit b7e493b4e4
(run trigger_post_build
after every successful spec) broke a lot of builders. The configs were
written to be called only after everything was done, but by calling it
after the first build (stage1 only), all the other artifacts we expected
did not yet exist.
Change post_build to pass the current set & spec name down so the config
can figure out what is available at that point in time.
440 lines
12 KiB
Bash
Executable file
440 lines
12 KiB
Bash
Executable file
#!/bin/bash
|
|
# Copyright 1999-2014 Gentoo Foundation
|
|
# Distributed under the terms of the GNU General Public License v2
|
|
|
|
CATALYST_CONFIG=/etc/catalyst/catalyst.conf
|
|
|
|
# Probe the default source dir from this script name.
|
|
REPO_DIR=$(dirname "$(dirname "$(realpath "$0")")")
|
|
|
|
# Set up defaults that config files can override if they want.
|
|
SUBARCH=$(uname -m)
|
|
EMAIL_TO="releng@gentoo.org,gentoo-releng-autobuilds@lists.gentoo.org"
|
|
# Use full hostname by default as Gentoo servers will reject short names.
|
|
EMAIL_FROM="catalyst@$(hostname -f)"
|
|
EMAIL_SUBJECT_PREPEND="[${SUBARCH}-auto]"
|
|
|
|
# Variables updated by command line arguments.
|
|
declare -a config_files
|
|
config_files=()
|
|
verbose=0
|
|
keep_tmpdir=0
|
|
testing=0
|
|
preclean=0
|
|
lastrun=0
|
|
lock_file=
|
|
|
|
usage() {
|
|
local msg=$1
|
|
|
|
if [ -n "${msg}" ]; then
|
|
printf "%b\n\n" "${msg}"
|
|
fi
|
|
|
|
cat <<EOH
|
|
Usage:
|
|
catalyst-auto [-c|--config <config>] [-v|--verbose] [-h|--help]
|
|
|
|
Options:
|
|
-c|--config Specifies the config file to use (required)
|
|
-C|--preclean Clean up loose artifacts from previous runs
|
|
-v|--verbose Send output of commands to console as well as log
|
|
-k|--keep-tmpdir Don't remove temp dir when build finishes
|
|
-t|--test Stop after mangling specs and copying files
|
|
--interval <days> Exit if last successful run was less than <days> ago
|
|
-l|--lock <file> File to grab a lock on to prevent multiple invocations
|
|
-h|--help Show this message and quit
|
|
|
|
EOH
|
|
}
|
|
|
|
send_email() {
|
|
local subject="${EMAIL_SUBJECT_PREPEND} $1"
|
|
local message=$2
|
|
local logfile=$3
|
|
local body
|
|
|
|
if [ -n "${logfile}" ]; then
|
|
body=$(printf '%b\n\n\n' "${message}"; tail -n 200 "${logfile}"; printf '\n\n\nFull build log at %s\n' "${logfile}")
|
|
else
|
|
body=${message}
|
|
fi
|
|
|
|
printf 'From: %s\r\nTo: %s\r\nSubject: %s\r\n\r\n%b' \
|
|
"${EMAIL_FROM}" "${EMAIL_TO}" "${subject}" "${body}" | \
|
|
/usr/sbin/sendmail -f "${EMAIL_FROM}" ${EMAIL_TO//,/ }
|
|
}
|
|
|
|
# Usage: run_cmd <logfile> <command to run>
|
|
run_cmd() {
|
|
local logfile="$1"
|
|
shift
|
|
|
|
if [ $verbose = 2 ]; then
|
|
echo "*** Running command: $*"
|
|
"$@" 2>&1 | tee -a "${logfile}"
|
|
elif [ $verbose = 1 ]; then
|
|
echo "*** Running command: $*"
|
|
"$@" &>> "${logfile}"
|
|
else
|
|
"$@" &>> "${logfile}"
|
|
fi
|
|
|
|
# If we used tee above, make sure we pass back up the command's error.
|
|
return ${PIPESTATUS[0]}
|
|
}
|
|
|
|
update_symlinks() {
|
|
# This is a skeleton function that you can override from the config file.
|
|
# It will be called by pre_build and after completing the build of a set
|
|
# to ensure the symlinks point to the latest built stages.
|
|
:
|
|
}
|
|
|
|
pre_build() {
|
|
# This is a skeleton function that you can override from the config file.
|
|
# It will be executed before the build is started.
|
|
update_symlinks
|
|
}
|
|
|
|
post_build() {
|
|
# This is a skeleton function that you can override from the config file.
|
|
# It will be executed after the build is successfully completed. You can
|
|
# use this to rsync the builds to another box
|
|
:
|
|
}
|
|
|
|
catalyst_var() {
|
|
# Extract a setting from the catalyst.conf.
|
|
local var=$1
|
|
(. "${CATALYST_CONFIG}"; echo "${!var}")
|
|
}
|
|
|
|
trigger_post_build() {
|
|
local set=$1 spec=$2
|
|
if ! run_cmd "${TMPDIR}/log/post_build.log" post_build "${set}" "${spec}"; then
|
|
send_email "Catalyst build error - post_build" "The post_build function failed" "${TMPDIR}/log/post_build.log"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
parse_args() {
|
|
local a
|
|
while [[ $# -gt 0 ]] ; do
|
|
a=$1
|
|
shift
|
|
case "${a}" in
|
|
-h|--help)
|
|
usage
|
|
exit 0
|
|
;;
|
|
-c|--config)
|
|
config_files+=("$1")
|
|
shift
|
|
;;
|
|
-v|--verbose)
|
|
verbose=$(($verbose+1))
|
|
;;
|
|
-k|--keep-tmpdir)
|
|
keep_tmpdir=1
|
|
;;
|
|
-t|--test)
|
|
testing=1
|
|
;;
|
|
-C|--preclean)
|
|
preclean=1
|
|
;;
|
|
--interval)
|
|
lastrun=$1
|
|
shift
|
|
;;
|
|
-l|--lock)
|
|
lock_file=$1
|
|
shift
|
|
;;
|
|
-*)
|
|
usage "ERROR: You have specified an invalid option: ${a}"
|
|
exit 1
|
|
;;
|
|
*)
|
|
usage "ERROR: This script takes no arguments: ${a}"
|
|
exit 1
|
|
;;
|
|
esac
|
|
done
|
|
}
|
|
|
|
# Let's get our own namespaces/etc... to avoid leaking crap.
|
|
containerize() {
|
|
# If we've already relaunched, nothing to do.
|
|
if [[ ${UNSHARE} == "true" ]] ; then
|
|
return
|
|
fi
|
|
|
|
# Most systems have unshare available, but just in case.
|
|
if type -P unshare >&/dev/null ; then
|
|
local uargs=()
|
|
# Probe the namespaces as some can be disabled (or we are not root).
|
|
unshare -m -- true >&/dev/null && uargs+=( -m )
|
|
unshare -u -- true >&/dev/null && uargs+=( -u )
|
|
unshare -i -- true >&/dev/null && uargs+=( -i )
|
|
unshare -p -- true >&/dev/null && uargs+=( -p -f --mount-proc )
|
|
# Re-exec ourselves in the new namespace.
|
|
UNSHARE=true exec unshare "${uargs[@]}" -- "$0" "$@"
|
|
fi
|
|
}
|
|
|
|
# Update the git repo if possible. It might modify this script which will probably
|
|
# make bash fail (since bash parses as it executes). So we have to safely re-exec
|
|
# the script whenever there's an update.
|
|
git_update() {
|
|
# If we've already relaunched, nothing to do.
|
|
if [[ ${GIT_UPDATE} == "true" ]] ; then
|
|
return
|
|
fi
|
|
|
|
pushd "${REPO_DIR}" >/dev/null
|
|
git fetch -q
|
|
revs=$(git rev-list HEAD..FETCH_HEAD)
|
|
popd >/dev/null
|
|
if [[ -n ${revs} ]] ; then
|
|
GIT_UPDATE=true exec bash -c '
|
|
repo_dir=$1 script=$2
|
|
shift 2
|
|
pushd "${repo_dir}" >/dev/null
|
|
git merge FETCH_HEAD || echo "${script}: WARNING: git repo is dirty"
|
|
popd >/dev/null
|
|
exec "${script}" "$@"
|
|
' -- "${REPO_DIR}" "$0" "$@"
|
|
fi
|
|
}
|
|
|
|
run_catalyst_commands() {
|
|
doneconfig=0
|
|
for config_file in "${config_files[@]}"; do
|
|
# Make sure all required values were specified.
|
|
if [[ -z "${config_file}" || ! -e "${config_file}" ]]; then
|
|
usage "ERROR: You must specify a valid config file to use: '$config_file' is not valid"
|
|
exit 1
|
|
fi
|
|
source "${config_file}"
|
|
doneconfig=1
|
|
done
|
|
if [[ ${doneconfig} == 0 ]]; then
|
|
usage "ERROR: You must specify at least one valid config file to use"
|
|
exit 1
|
|
fi
|
|
|
|
# Some configs will set this explicitly, so don't clobber it.
|
|
: ${BUILD_SRCDIR_BASE:=$(catalyst_var storedir)}
|
|
|
|
# See if we had a recent success.
|
|
if [[ ${lastrun} != 0 ]]; then
|
|
last_success_file="${BUILD_SRCDIR_BASE}/.last_success"
|
|
delay=$(( lastrun * 24 * 60 * 60 ))
|
|
last_success=$(head -1 "${last_success_file}" 2>/dev/null || echo 0)
|
|
if [[ $(date +%s) -lt $(( last_success + delay )) ]]; then
|
|
exit 0
|
|
fi
|
|
fi
|
|
|
|
DATESTAMP=$(date -u +%Y%m%d)
|
|
TIMESTAMP=$(date -u +%Y%m%dT%H%M%SZ)
|
|
TMPDIR=$(mktemp -d --tmpdir="${TMP_PATH:-/tmp}" "catalyst-auto.${TIMESTAMP}.XXXXXX")
|
|
|
|
# Nuke any previous tmpdirs to keep them from accumulating.
|
|
if [[ ${preclean} == 1 ]]; then
|
|
rm -rf "${TMPDIR%.??????}".*
|
|
mkdir "${TMPDIR}"
|
|
fi
|
|
|
|
if [[ ${verbose} -ge 1 ]]; then
|
|
echo "TMPDIR = ${TMPDIR}"
|
|
echo "DATESTAMP = ${DATESTAMP}"
|
|
echo "TIMESTAMP = ${TIMESTAMP}"
|
|
fi
|
|
|
|
if ! mkdir -p "${TMPDIR}"/{specs,kconfig,log}; then
|
|
echo "Couldn't create tempdirs!"
|
|
exit 1
|
|
fi
|
|
|
|
if ! run_cmd "${TMPDIR}/log/pre_build.log" pre_build; then
|
|
send_email "Catalyst build error - pre_build" "The pre_build function failed" "${TMPDIR}/log/pre_build.log"
|
|
exit 1
|
|
fi
|
|
|
|
cd "${SPECS_DIR}" || exit 1
|
|
|
|
for a in "" ${SETS}; do
|
|
if [[ -z "${a}" ]]; then
|
|
specs_var="SPECS"
|
|
optional_specs_var="OPTIONAL_SPECS"
|
|
else
|
|
specs_var="SET_${a}_SPECS"
|
|
optional_specs_var="SET_${a}_OPTIONAL_SPECS"
|
|
fi
|
|
|
|
for i in ${!specs_var} ${!optional_specs_var}; do
|
|
cp --parents "${i}" "${TMPDIR}"/specs/
|
|
done
|
|
done
|
|
|
|
find "${KCONFIG_DIR}" -type f -exec cp {} "${TMPDIR}"/kconfig \;
|
|
|
|
cd "${TMPDIR}/specs" || exit 1
|
|
|
|
# Fix up specs with datestamp
|
|
for i in $(find -name '*.spec'); do
|
|
# Grab current version_stamp and source_subpath
|
|
old_version_stamp=$(grep version_stamp "${i}" | sed -e 's|^version_stamp: *||')
|
|
old_source_subpath=$(grep source_subpath "${i}" | sed -e 's|^source_subpath: *||')
|
|
|
|
new_version_stamp=$(echo "${old_version_stamp}" | sed -e "s|^\(.*-\)\?.*$|\1${TIMESTAMP}|")
|
|
new_source_subpath=$(echo "${old_source_subpath}" | sed -e "s|${old_version_stamp}|${new_version_stamp}|")
|
|
|
|
sed -i "s|^version_stamp:.*$|version_stamp: ${new_version_stamp}|" "${i}"
|
|
sed -i "s|^snapshot:.*$|snapshot: ${TIMESTAMP}|" "${i}"
|
|
|
|
# We don't want to mangle the source_subpath for our stage1 spec
|
|
if ! grep -q '^target: *stage[14]$' "${i}"; then
|
|
sed -i "s|^source_subpath:.*$|source_subpath: ${new_source_subpath}|" "${i}"
|
|
fi
|
|
|
|
sed -i "/^livecd\/iso/s|${old_version_stamp}|${new_version_stamp}|" "${i}"
|
|
sed -i "/^livecd\/volid/s|${old_version_stamp}|${new_version_stamp}|" "${i}"
|
|
|
|
kconfig_lines=$(grep '^boot/kernel/[^/]\+/config:' "${i}")
|
|
if [[ -n ${kconfig_lines} ]]; then
|
|
echo "${kconfig_lines}" | while read line; do
|
|
key=$(echo "${line}" | cut -d: -f1)
|
|
filename=$(basename $(echo "${line}" | cut -d: -f2))
|
|
sed -i "s|^${key}:.*\$|${key}: ${TMPDIR}/kconfig/${filename}|" "${i}"
|
|
done
|
|
fi
|
|
|
|
# Expand vars that the spec expects us to.
|
|
sed -i \
|
|
-e "s:@DATESTAMP@:${DATESTAMP}:g" \
|
|
-e "s:@TIMESTAMP@:${TIMESTAMP}:g" \
|
|
-e "s:@REPO_DIR@:${REPO_DIR}:g" \
|
|
"${i}"
|
|
done
|
|
|
|
if [[ ${testing} == 1 ]]; then
|
|
echo "Exiting due to --test"
|
|
exit
|
|
fi
|
|
|
|
if [[ ${preclean} == 1 ]]; then
|
|
snapshot_cache=$(catalyst_var snapshot_cache)
|
|
if [[ -z ${snapshot_cache} ]]; then
|
|
echo "error: snapshot_cache not set in config file"
|
|
exit 1
|
|
fi
|
|
pushd "${BUILD_SRCDIR_BASE}" >/dev/null || exit 1
|
|
rm -rf --one-file-system \
|
|
kerncache packages snapshots tmp "${snapshot_cache}"/*
|
|
popd >/dev/null
|
|
fi
|
|
|
|
# Create snapshot
|
|
if ! run_cmd "${TMPDIR}/log/snapshot.log" catalyst -c "${CATALYST_CONFIG}" -s "${TIMESTAMP}"; then
|
|
send_email "Catalyst build error - snapshot" "" "${TMPDIR}/log/snapshot.log"
|
|
exit 1
|
|
fi
|
|
|
|
build_failure=0
|
|
|
|
timeprefix=()
|
|
which time >/dev/null && timeprefix=( "time" )
|
|
|
|
for a in "" ${SETS}; do
|
|
if [[ -z ${a} ]]; then
|
|
specs_var="SPECS"
|
|
optional_specs_var="OPTIONAL_SPECS"
|
|
else
|
|
specs_var="SET_${a}_SPECS"
|
|
optional_specs_var="SET_${a}_OPTIONAL_SPECS"
|
|
fi
|
|
|
|
for i in ${!specs_var}; do
|
|
LOGFILE="${TMPDIR}/log/$(echo "${i}" | sed -e 's:/:_:' -e 's:\.spec$::').log"
|
|
run_cmd "${LOGFILE}" "${timeprefix[@]}" catalyst -a -c "${CATALYST_CONFIG}" -f "${i}"
|
|
if [[ $? != 0 ]]; then
|
|
build_failure=1
|
|
send_email "Catalyst fatal build error - ${i}" "" "${LOGFILE}"
|
|
continue 2
|
|
else
|
|
trigger_post_build "${a}" "${i}"
|
|
fi
|
|
done
|
|
|
|
for i in ${!optional_specs_var}; do
|
|
LOGFILE="${TMPDIR}/log/$(echo "${i}" | sed -e 's:/:_:' -e 's:\.spec$::').log"
|
|
run_cmd "${LOGFILE}" "${timeprefix[@]}" catalyst -a -c "${CATALYST_CONFIG}" -f "${i}"
|
|
if [[ $? != 0 ]]; then
|
|
build_failure=1
|
|
send_email "Catalyst non-fatal build error - ${i}" "" "${LOGFILE}"
|
|
break
|
|
else
|
|
trigger_post_build "${a}" "${i}"
|
|
fi
|
|
done
|
|
|
|
for i in ${!specs_var} ${!optional_specs_var}; do
|
|
LOGFILE="${TMPDIR}/log/$(echo "${i}" | sed -e 's:/:_:' -e 's:\.spec$::')_purge.log"
|
|
run_cmd "${LOGFILE}" "${timeprefix[@]}" catalyst --purgetmponly -c "${CATALYST_CONFIG}" -f "${i}"
|
|
done
|
|
|
|
update_symlinks
|
|
done
|
|
|
|
trigger_post_build
|
|
|
|
if [[ ${build_failure} == 0 ]]; then
|
|
if [[ ${lastrun} != 0 ]]; then
|
|
stamp=$(date)
|
|
(date -d"${stamp}" +%s; echo "${stamp}") >"${last_success_file}"
|
|
fi
|
|
|
|
send_email "Catalyst build success" "Build process complete."
|
|
|
|
if [[ ${keep_tmpdir} == 0 ]]; then
|
|
if ! rm -rf "${TMPDIR}"; then
|
|
echo "Could not remove tmpdir ${TMPDIR}!"
|
|
exit 1
|
|
fi
|
|
fi
|
|
else
|
|
send_email "Catalyst build complete, but with errors" "Build process has completed, but there were errors. Please consult previous emails to determine the problem."
|
|
fi
|
|
}
|
|
|
|
main() {
|
|
# Set pipefail so that run_cmd returns the right value in $?.
|
|
set -o pipefail
|
|
|
|
# Parse user arguments before we try doing container logic.
|
|
parse_args "$@"
|
|
|
|
# Update the release git dir if possible.
|
|
git_update "$@"
|
|
|
|
# Try to isolate ourselves from the rest of the system.
|
|
containerize "$@"
|
|
|
|
(
|
|
if [[ -n ${lock_file} ]]; then
|
|
if ! flock -n 9; then
|
|
echo "catalyst-auto already running"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
run_catalyst_commands
|
|
) 9>"${lock_file:-/dev/null}"
|
|
}
|
|
|
|
main "$@"
|