2008-09-04 01:18:17 +00:00
#!/bin/bash
2014-09-10 19:52:48 +00:00
# Copyright 1999-2014 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
2008-09-04 01:18:17 +00:00
2014-09-10 19:54:16 +00:00
# First let's get our own namespaces to avoid leaking crap.
2014-01-22 05:58:29 +00:00
if [[ -z ${UNSHARE} ]] ; then
if type -P unshare >&/dev/null ; then
2014-09-10 19:54:16 +00:00
uargs=()
# Probe the namespaces as some can be disabled (or we are not root).
unshare -m -- true >&/dev/null && uargs+=( -m )
unshare -u -- true >&/dev/null && uargs+=( -u )
unshare -i -- true >&/dev/null && uargs+=( -i )
unshare -p -- true >&/dev/null && uargs+=( -p -f --mount-proc )
# Re-exec ourselves in the new namespace.
UNSHARE=true exec unshare "${uargs[@]}" -- "$0" "$@"
2014-01-22 05:58:29 +00:00
fi
fi
unset UNSHARE
2016-03-21 05:21:05 +00:00
CATALYST_CONFIG=/etc/catalyst/catalyst.conf
2016-01-26 07:17:34 +00:00
declare -a config_files
config_files=()
2008-09-04 01:19:35 +00:00
verbose=0
2008-09-28 20:15:21 +00:00
keep_tmpdir=0
2009-03-13 12:34:18 +00:00
testing=0
2016-03-21 05:27:34 +00:00
preclean=0
2016-03-21 05:34:12 +00:00
lastrun=0
2016-03-23 18:20:52 +00:00
lock_file=
2008-09-04 01:19:35 +00:00
2014-04-15 11:07:03 +00:00
# Set pipefail so that run_cmd returns the right value in $?
set -o pipefail
2008-09-04 01:18:17 +00:00
usage() {
2014-09-10 19:51:20 +00:00
local msg=$1
2008-09-04 01:18:17 +00:00
if [ -n "${msg}" ]; then
2014-09-10 19:51:20 +00:00
printf "%b\n\n" "${msg}"
2008-09-04 01:18:17 +00:00
fi
cat <<EOH
Usage:
2011-12-02 05:15:46 +00:00
stage_build [-c|--config <config>] [-v|--verbose] [-h|--help]
2008-09-04 01:18:17 +00:00
Options:
-c|--config Specifies the config file to use (required)
2016-03-21 05:27:34 +00:00
-C|--preclean Clean up loose artifacts from previous runs
2011-12-02 05:15:46 +00:00
-v|--verbose Send output of commands to console as well as log
2009-01-02 01:50:52 +00:00
-k|--keep-tmpdir Don't remove temp dir when build finishes
2009-03-13 12:34:18 +00:00
-t|--test Stop after mangling specs and copying files
2016-03-21 05:34:12 +00:00
--interval <days> Exit if last successful run was less than <days> ago
2016-03-23 18:20:52 +00:00
-l|--lock <file> File to grab a lock on to prevent multiple invocations
2008-09-04 01:18:17 +00:00
-h|--help Show this message and quit
EOH
}
send_email() {
2016-05-24 02:02:22 +00:00
local subject="${EMAIL_SUBJECT_PREPEND} $1"
local message=$2
local logfile=$3
2008-09-04 01:18:17 +00:00
2008-09-28 20:15:21 +00:00
if [ -n "${logfile}" ]; then
2016-05-24 02:00:31 +00:00
body=$(printf '%b\n\n\n' "${message}"; tail -n 200 "${logfile}"; printf '\n\n\nFull build log at %s\n' "${logfile}")
2008-09-28 20:15:21 +00:00
else
body=${message}
fi
2016-05-24 02:00:31 +00:00
printf 'From: %s\r\nTo: %s\r\nSubject: %s\r\n\r\n%b' \
"${EMAIL_FROM}" "${EMAIL_TO}" "${subject}" "${body}" | \
/usr/sbin/sendmail -f "${EMAIL_FROM}" ${EMAIL_TO//,/ }
2008-09-04 01:18:17 +00:00
}
2016-12-09 04:51:29 +00:00
# Usage: run_cmd <logfile> <command to run>
2008-09-04 01:18:17 +00:00
run_cmd() {
2016-12-09 04:51:29 +00:00
local logfile="$1"
shift
2008-09-04 01:18:17 +00:00
if [ $verbose = 1 ]; then
2016-12-09 04:51:29 +00:00
echo "*** Running command: $*"
"$@" 2>&1 | tee "${logfile}"
2008-09-04 01:18:17 +00:00
else
2016-12-09 04:51:29 +00:00
"$@" &> "${logfile}"
2008-09-04 01:18:17 +00:00
fi
2016-12-09 04:53:03 +00:00
# If we used tee above, make sure we pass back up the command's error.
return ${PIPESTATUS[0]}
2008-09-04 01:18:17 +00:00
}
2016-05-05 10:07:20 +00:00
update_symlinks() {
# This is a skeleton function that you can override from the config file.
# It will be called by pre_build and after completing the build of a set
# to ensure the symlinks point to the latest built stages.
2016-12-09 04:36:17 +00:00
:
2016-05-05 10:07:20 +00:00
}
2008-09-28 04:31:49 +00:00
pre_build() {
# This is a skeleton function that you can override from the config file.
# It will be executed before the build is started. You can use this to
2012-02-13 01:10:01 +00:00
# update the checkout of the releng repo
2016-12-09 04:36:17 +00:00
:
2008-09-28 04:31:49 +00:00
}
post_build() {
# This is a skeleton function that you can override from the config file.
# It will be executed after the build is successfully completed. You can
# use this to rsync the builds to another box
2016-12-09 04:36:17 +00:00
:
2008-09-28 04:31:49 +00:00
}
2016-03-21 05:21:05 +00:00
catalyst_var() {
# Extract a setting from the catalyst.conf.
local var=$1
(. "${CATALYST_CONFIG}"; echo "${!var}")
}
2008-09-04 01:18:17 +00:00
# Parse args
while [ ${#} -gt 0 ]
do
a=${1}
shift
case "${a}" in
-h|--help)
usage
exit 0
;;
-c|--config)
2016-01-26 07:17:34 +00:00
config_files+=("$1")
2008-09-04 01:18:17 +00:00
shift
;;
2011-12-02 05:15:46 +00:00
-v|--verbose)
2008-09-04 01:18:17 +00:00
verbose=1
;;
2008-09-28 20:15:21 +00:00
-k|--keep-tmpdir)
keep_tmpdir=1
;;
2009-03-13 12:34:18 +00:00
-t|--test)
testing=1
;;
2016-03-21 05:27:34 +00:00
-C|--preclean)
preclean=1
;;
2016-03-21 05:34:12 +00:00
--interval)
lastrun=$1
shift
;;
2016-03-23 18:20:52 +00:00
-l|--lock)
lock_file=$1
shift
;;
2008-09-04 01:18:17 +00:00
-*)
2014-09-10 19:51:20 +00:00
usage "ERROR: You have specified an invalid option: ${a}"
2008-09-04 01:18:17 +00:00
exit 1
;;
2016-12-09 04:38:18 +00:00
*)
usage "ERROR: This script takes no arguments: ${a}"
exit 1
;;
2008-09-04 01:18:17 +00:00
esac
done
2016-03-23 18:20:52 +00:00
(
if [[ -n ${lock_file} ]]; then
if ! flock -n 9; then
echo "catalyst-auto already running"
exit 1
fi
fi
2016-05-20 02:16:14 +00:00
# Probe the default source dir from this script name.
REPO_DIR=$(dirname "$(dirname "$(realpath "$0")")")
2016-01-26 07:17:34 +00:00
2016-03-27 05:31:30 +00:00
# Set up defaults that config files can override if they want.
SUBARCH=$(uname -m)
EMAIL_TO="releng@gentoo.org,gentoo-releng-autobuilds@lists.gentoo.org"
EMAIL_FROM="catalyst@${HOSTNAME:-$(hostname)}"
EMAIL_SUBJECT_PREPEND="[${SUBARCH}-auto]"
2016-01-26 07:17:34 +00:00
doneconfig=0
2016-12-09 04:45:55 +00:00
for config_file in "${config_files[@]}"; do
2016-01-26 07:17:34 +00:00
# Make sure all required values were specified
if [ -z "${config_file}" -o ! -e "${config_file}" ]; then
usage "ERROR: You must specify a valid config file to use: '$config_file' is not valid"
exit 1
fi
2016-12-09 04:45:55 +00:00
source "${config_file}"
2016-01-26 07:17:34 +00:00
doneconfig=1
done
if [[ $doneconfig -eq 0 ]]; then
usage "ERROR: You must specify at least one valid config file to use"
exit 1
fi
2016-03-24 05:25:28 +00:00
# Some configs will set this explicitly, so don't clobber it.
: ${BUILD_SRCDIR_BASE:=$(catalyst_var storedir)}
2008-09-04 01:18:17 +00:00
2016-03-21 05:34:12 +00:00
# See if we had a recent success.
if [[ ${lastrun} -ne 0 ]]; then
last_success_file="${BUILD_SRCDIR_BASE}/.last_success"
delay=$(( lastrun * 24 * 60 * 60 ))
last_success=$(head -1 "${last_success_file}" 2>/dev/null || echo 0)
if [[ $(date +%s) -lt $(( last_success + delay )) ]]; then
exit 0
fi
fi
2008-09-04 02:11:48 +00:00
DATESTAMP=$(date +%Y%m%d)
2016-12-09 04:45:55 +00:00
TMPDIR=$(mktemp -d --tmpdir="${TMP_PATH:-/tmp}" "catalyst-auto.${DATESTAMP}.XXXXXX")
2008-09-04 02:11:48 +00:00
2016-03-21 05:27:34 +00:00
# Nuke any previous tmpdirs to keep them from accumulating.
if [[ ${preclean} -eq 1 ]]; then
rm -rf "${TMPDIR%.??????}".*
mkdir "${TMPDIR}"
fi
2008-09-04 02:11:48 +00:00
if [ ${verbose} = 1 ]; then
echo "TMPDIR = ${TMPDIR}"
echo "DATESTAMP = ${DATESTAMP}"
fi
2008-09-04 01:18:17 +00:00
2014-09-10 19:52:48 +00:00
if ! mkdir -p "${TMPDIR}"/{specs,kconfig,log}; then
echo "Couldn't create tempdirs!"
exit 1
2008-09-04 01:18:17 +00:00
fi
2016-12-09 04:51:29 +00:00
if ! run_cmd "${TMPDIR}/log/pre_build.log" pre_build; then
2016-03-22 16:46:36 +00:00
send_email "Catalyst build error - pre_build" "The pre_build function failed" "${TMPDIR}/log/pre_build.log"
2008-09-28 04:39:45 +00:00
exit 1
fi
2016-12-09 04:45:55 +00:00
cd "${SPECS_DIR}" || exit 1
2008-09-04 01:18:17 +00:00
2009-06-27 19:09:30 +00:00
for a in "" ${SETS}; do
if [ -z "${a}" ]; then
specs_var="SPECS"
optional_specs_var="OPTIONAL_SPECS"
else
specs_var="SET_${a}_SPECS"
optional_specs_var="SET_${a}_OPTIONAL_SPECS"
fi
for i in ${!specs_var} ${!optional_specs_var}; do
2016-12-09 04:45:55 +00:00
cp --parents "${i}" "${TMPDIR}"/specs/
2009-06-27 19:09:30 +00:00
done
2008-09-04 01:18:17 +00:00
done
2016-12-09 04:45:55 +00:00
find "${KCONFIG_DIR}" -type f -exec cp {} "${TMPDIR}"/kconfig \;
2008-09-04 12:44:33 +00:00
2016-12-09 04:45:55 +00:00
cd "${TMPDIR}/specs" || exit 1
2008-09-04 01:18:17 +00:00
2008-09-04 01:54:15 +00:00
# Fix up specs with datestamp
for i in $(find -name '*.spec'); do
# Grab current version_stamp and source_subpath
2016-12-09 04:45:55 +00:00
old_version_stamp=$(grep version_stamp "${i}" | sed -e 's|^version_stamp: *||')
old_source_subpath=$(grep source_subpath "${i}" | sed -e 's|^source_subpath: *||')
2008-09-04 01:54:15 +00:00
2016-12-09 04:45:55 +00:00
new_version_stamp=$(echo "${old_version_stamp}" | sed -e "s|^\(.*-\)\?.*$|\1${DATESTAMP}|")
new_source_subpath=$(echo "${old_source_subpath}" | sed -e "s|${old_version_stamp}|${new_version_stamp}|")
2008-12-02 23:03:57 +00:00
2016-12-09 04:45:55 +00:00
sed -i "s|^version_stamp:.*$|version_stamp: ${new_version_stamp}|" "${i}"
sed -i "s|^snapshot:.*$|snapshot: ${DATESTAMP}|" "${i}"
2008-11-05 01:47:53 +00:00
# We don't want to mangle the source_subpath for our stage1 spec
2016-12-09 04:45:55 +00:00
if ! grep -q '^target: *stage[14]$' "${i}"; then
sed -i "s|^source_subpath:.*$|source_subpath: ${new_source_subpath}|" "${i}"
2008-09-04 01:54:15 +00:00
fi
2008-11-05 01:47:53 +00:00
2016-12-09 04:45:55 +00:00
sed -i "/^livecd\/iso/s|${old_version_stamp}|${new_version_stamp}|" "${i}"
sed -i "/^livecd\/volid/s|${old_version_stamp}|${new_version_stamp}|" "${i}"
2008-09-04 12:44:33 +00:00
2016-12-09 04:45:55 +00:00
kconfig_lines=$(grep '^boot/kernel/[^/]\+/config:' "${i}")
2009-03-13 12:51:55 +00:00
if [ -n "${kconfig_lines}" ]; then
echo "${kconfig_lines}" | while read line; do
2009-03-02 13:15:10 +00:00
key=$(echo "${line}" | cut -d: -f1)
filename=$(basename $(echo "${line}" | cut -d: -f2))
2016-12-09 04:45:55 +00:00
sed -i "s|^${key}:.*\$|${key}: ${TMPDIR}/kconfig/${filename}|" "${i}"
2009-03-02 13:15:10 +00:00
done
2008-09-04 12:44:33 +00:00
fi
2016-03-20 20:14:42 +00:00
# Expand vars that the spec expects us to.
sed -i \
-e "s:@DATESTAMP@:${DATESTAMP}:g" \
2016-04-26 22:27:56 +00:00
-e "s:@REPO_DIR@:${REPO_DIR}:g" \
2016-03-20 20:14:42 +00:00
"${i}"
2008-09-04 01:54:15 +00:00
done
2009-03-13 12:34:18 +00:00
if [ "${testing}" -eq 1 ]; then
echo "Exiting due to --test"
exit
fi
2016-03-21 05:27:34 +00:00
if [[ ${preclean} -eq 1 ]]; then
snapshot_cache=$(catalyst_var snapshot_cache)
if [[ -z ${snapshot_cache} ]]; then
echo "error: snapshot_cache not set in config file"
exit 1
fi
pushd "${BUILD_SRCDIR_BASE}" >/dev/null || exit 1
rm -rf --one-file-system \
kerncache packages snapshots tmp "${snapshot_cache}"/*
popd >/dev/null
fi
2008-09-04 01:54:15 +00:00
# Create snapshot
2016-12-09 04:51:29 +00:00
if ! run_cmd "${TMPDIR}/log/snapshot.log" catalyst -c "${CATALYST_CONFIG}" -s "${DATESTAMP}"; then
2008-09-28 20:15:21 +00:00
send_email "Catalyst build error - snapshot" "" "${TMPDIR}/log/snapshot.log"
2008-09-28 04:31:49 +00:00
exit 1
2008-09-04 01:54:15 +00:00
fi
2009-11-02 13:31:26 +00:00
build_failure=0
2016-12-09 04:51:29 +00:00
timeprefix=()
which time >/dev/null && timeprefix=( "time" )
2016-05-30 16:13:32 +00:00
2009-06-27 19:09:30 +00:00
for a in "" ${SETS}; do
if [ -z "${a}" ]; then
specs_var="SPECS"
optional_specs_var="OPTIONAL_SPECS"
else
specs_var="SET_${a}_SPECS"
optional_specs_var="SET_${a}_OPTIONAL_SPECS"
2008-11-29 02:15:02 +00:00
fi
2009-06-27 19:09:30 +00:00
for i in ${!specs_var}; do
LOGFILE="${TMPDIR}/log/$(echo "${i}" | sed -e 's:/:_:' -e 's:\.spec$::').log"
2016-12-09 04:51:29 +00:00
run_cmd "${LOGFILE}" "${timeprefix[@]}" catalyst -a -p -c "${CATALYST_CONFIG}" -f "${i}"
2009-06-27 19:09:30 +00:00
if [ $? != 0 ]; then
2009-11-02 13:31:26 +00:00
build_failure=1
2009-06-27 19:09:30 +00:00
send_email "Catalyst fatal build error - ${i}" "" "${LOGFILE}"
continue 2
fi
done
for i in ${!optional_specs_var}; do
LOGFILE="${TMPDIR}/log/$(echo "${i}" | sed -e 's:/:_:' -e 's:\.spec$::').log"
2016-12-09 04:51:29 +00:00
run_cmd "${LOGFILE}" "${timeprefix[@]}" catalyst -a -p -c "${CATALYST_CONFIG}" -f "${i}"
2009-06-27 19:09:30 +00:00
if [ $? != 0 ]; then
2009-11-02 13:31:26 +00:00
build_failure=1
2009-06-27 19:09:30 +00:00
send_email "Catalyst non-fatal build error - ${i}" "" "${LOGFILE}"
break
fi
done
for i in ${!specs_var} ${!optional_specs_var}; do
LOGFILE="${TMPDIR}/log/$(echo "${i}" | sed -e 's:/:_:' -e 's:\.spec$::')_purge.log"
2016-12-09 04:51:29 +00:00
run_cmd "${LOGFILE}" "${timeprefix[@]}" catalyst -P -c "${CATALYST_CONFIG}" -f "${i}"
2009-06-27 19:09:30 +00:00
done
2016-05-05 10:07:20 +00:00
update_symlinks
2008-12-22 04:52:31 +00:00
done
2008-11-29 02:15:02 +00:00
2016-12-09 04:51:29 +00:00
if ! run_cmd "${TMPDIR}/log/post_build.log" post_build; then
2016-03-22 16:46:36 +00:00
send_email "Catalyst build error - post_build" "The post_build function failed" "${TMPDIR}/log/post_build.log"
2008-09-28 04:31:49 +00:00
exit 1
fi
2009-11-02 13:31:26 +00:00
if [ ${build_failure} = 0 ]; then
2016-03-21 05:34:12 +00:00
if [[ ${lastrun} -ne 0 ]]; then
stamp=$(date)
(date -d"${stamp}" +%s; echo "${stamp}") >"${last_success_file}"
fi
2009-11-02 13:31:26 +00:00
send_email "Catalyst build success" "Build process complete."
2008-09-04 02:07:52 +00:00
2009-11-02 13:31:26 +00:00
if [ "${keep_tmpdir}" = 0 ]; then
if ! rm -rf "${TMPDIR}"; then
echo "Could not remove tmpdir ${TMPDIR}!"
exit 1
fi
2008-09-28 20:15:21 +00:00
fi
2009-11-02 13:31:26 +00:00
else
send_email "Catalyst build complete, but with errors" "Build process has completed, but there were errors. Please consult previous emails to determine the problem."
2008-09-04 02:07:52 +00:00
fi
2016-03-23 18:20:52 +00:00
) 9>"${lock_file:-/dev/null}"