gentoo-asahi-releng/tools/catalyst-auto

#!/bin/bash
# Copyright 1999-2014 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2

# First let's get our own namespaces to avoid leaking crap.
if [[ -z ${UNSHARE} ]] ; then
  if type -P unshare >&/dev/null ; then
    uargs=()
    # Probe the namespaces as some can be disabled (or we are not root).
    unshare -m -- true >&/dev/null && uargs+=( -m )
    unshare -u -- true >&/dev/null && uargs+=( -u )
    unshare -i -- true >&/dev/null && uargs+=( -i )
    unshare -p -- true >&/dev/null && uargs+=( -p -f --mount-proc )
    # Re-exec ourselves in the new namespace.
    UNSHARE=true exec unshare "${uargs[@]}" -- "$0" "$@"
  fi
fi
unset UNSHARE

CATALYST_CONFIG=/etc/catalyst/catalyst.conf

declare -a config_files
config_files=()
verbose=0
keep_tmpdir=0
testing=0
preclean=0
lastrun=0
lock_file=

# Set pipefail so that run_cmd returns the right value in $?
set -o pipefail

usage() {
  local msg=$1

  if [ -n "${msg}" ]; then
    printf "%b\n\n" "${msg}"
  fi

  cat <<EOH
Usage:
  stage_build [-c|--config <config>] [-v|--verbose] [-h|--help]

Options:
  -c|--config         Specifies the config file to use (required)
  -C|--preclean       Clean up loose artifacts from previous runs
  -v|--verbose        Send output of commands to console as well as log
  -k|--keep-tmpdir    Don't remove temp dir when build finishes
  -t|--test           Stop after mangling specs and copying files
  --interval <days>   Exit if last successful run was less than <days> ago
  -l|--lock <file>    File to grab a lock on to prevent multiple invocations
  -h|--help           Show this message and quit

EOH
}

send_email() {
  subject="${EMAIL_SUBJECT_PREPEND} $1"
  message=$2
  logfile=$3

  if [ -n "${logfile}" ]; then
    body=$(echo -e "${message}\n\n"; tail -n 200 "${logfile}"; echo -e "\n\n\nFull build log at ${logfile}")
  else
    body=${message}
  fi

  echo -e "From: ${EMAIL_FROM}\r\nTo: ${EMAIL_TO}\r\nSubject: ${subject}\r\n\r\n${body}" | /usr/sbin/sendmail -f ${EMAIL_FROM} ${EMAIL_TO//,/ }
}

run_cmd() {
  cmd=$1
  logfile=$2

  if [ $verbose = 1 ]; then
    echo "*** Running command: ${cmd}"
    ${cmd} 2>&1 | tee ${logfile}
  else
    ${cmd} &> ${logfile}
  fi
}

pre_build() {
  # This is a skeleton function that you can override from the config file.
  # It will be executed before the build is started. You can use this to
  # update the checkout of the releng repo
  local foo=bar
}

post_build() {
  # This is a skeleton function that you can override from the config file.
  # It will be executed after the build is successfully completed. You can
  # use this to rsync the builds to another box
  local foo=bar
}

catalyst_var() {
  # Extract a setting from the catalyst.conf.
  local var=$1
  (. "${CATALYST_CONFIG}"; echo "${!var}")
}

# Parse args
params=${#}
while [ ${#} -gt 0 ]
do
  a=${1}
  shift
  case "${a}" in
    -h|--help)
      usage
      exit 0
      ;;
    -c|--config)
      config_files+=("$1")
      shift
      ;;
    -v|--verbose)
      verbose=1
      ;;
    -k|--keep-tmpdir)
      keep_tmpdir=1
      ;;
    -t|--test)
      testing=1
      ;;
    -C|--preclean)
      preclean=1
      ;;
    --interval)
      lastrun=$1
      shift
      ;;
    -l|--lock)
      lock_file=$1
      shift
      ;;
    -*)
      usage "ERROR: You have specified an invalid option: ${a}"
      exit 1
      ;;
    esac
done

(

if [[ -n ${lock_file} ]]; then
  if ! flock -n 9; then
    echo "catalyst-auto already running"
    exit 1
  fi
fi

# Probe the default gitdir from this script name.
GITDIR=$(dirname "$(dirname "$(realpath "$0")")")

# Set up defaults that config files can override if they want.
SUBARCH=$(uname -m)
EMAIL_TO="releng@gentoo.org,gentoo-releng-autobuilds@lists.gentoo.org"
EMAIL_FROM="catalyst@${HOSTNAME:-$(hostname)}"
EMAIL_SUBJECT_PREPEND="[${SUBARCH}-auto]"

doneconfig=0
for config_file in ${config_files[@]}; do
    # Make sure all required values were specified
    if [ -z "${config_file}" -o ! -e "${config_file}" ]; then
      usage "ERROR: You must specify a valid config file to use: '$config_file' is not valid"
      exit 1
    fi
    source ${config_file}
    doneconfig=1
done
if [[ $doneconfig -eq 0 ]]; then
      usage "ERROR: You must specify at least one valid config file to use"
      exit 1
fi

# Some configs will set this explicitly, so don't clobber it.
: ${BUILD_SRCDIR_BASE:=$(catalyst_var storedir)}

# See if we had a recent success.
if [[ ${lastrun} -ne 0 ]]; then
  last_success_file="${BUILD_SRCDIR_BASE}/.last_success"
  delay=$(( lastrun * 24 * 60 * 60 ))
  last_success=$(head -1 "${last_success_file}" 2>/dev/null || echo 0)
  if [[ $(date +%s) -lt $(( last_success + delay )) ]]; then
    exit 0
  fi
fi

TMPDIR=$(mktemp -d --tmpdir="${TMP_PATH:-/tmp}" catalyst-auto.XXXXXX)
DATESTAMP=$(date +%Y%m%d)

# Nuke any previous tmpdirs to keep them from accumulating.
if [[ ${preclean} -eq 1 ]]; then
  rm -rf "${TMPDIR%.??????}".*
  mkdir "${TMPDIR}"
fi

if [ ${verbose} = 1 ]; then
  echo "TMPDIR = ${TMPDIR}"
  echo "DATESTAMP = ${DATESTAMP}"
fi

if ! mkdir -p "${TMPDIR}"/{specs,kconfig,log}; then
  echo "Couldn't create tempdirs!"
  exit 1
fi

if ! run_cmd "pre_build" "${TMPDIR}/log/pre_build.log"; then
  send_email "Catalyst build error - pre_build" "The pre_build function failed" "${TMPDIR}/log/pre_build.log"
  exit 1
fi

cd ${SPECS_DIR}

for a in "" ${SETS}; do
  if [ -z "${a}" ]; then
    specs_var="SPECS"
    optional_specs_var="OPTIONAL_SPECS"
  else
    specs_var="SET_${a}_SPECS"
    optional_specs_var="SET_${a}_OPTIONAL_SPECS"
  fi

  for i in ${!specs_var} ${!optional_specs_var}; do
    cp --parents ${i} ${TMPDIR}/specs/
  done
done

find ${KCONFIG_DIR} -type f -exec cp {} ${TMPDIR}/kconfig \;

cd ${TMPDIR}/specs

# Fix up specs with datestamp
for i in $(find -name '*.spec'); do
  # Grab current version_stamp and source_subpath
  old_version_stamp=$(grep version_stamp ${i} | sed -e 's|^version_stamp: *||')
  old_source_subpath=$(grep source_subpath ${i} | sed -e 's|^source_subpath: *||')

  new_version_stamp=$(echo "${old_version_stamp}" | sed -e 's|^\(.*-\)\?.*$|\1'${DATESTAMP}'|')
  new_source_subpath=$(echo "${old_source_subpath}" | sed -e 's|'${old_version_stamp}'|'${new_version_stamp}'|')

  sed -i 's|^version_stamp:.*$|version_stamp: '${new_version_stamp}'|' ${i}
  sed -i 's|^snapshot:.*$|snapshot: '${DATESTAMP}'|' ${i}

  # We don't want to mangle the source_subpath for our stage1 spec
  if ! grep -q '^target: *stage1$' ${i}; then
    sed -i 's|^source_subpath:.*$|source_subpath: '${new_source_subpath}'|' ${i}
  fi

  sed -i '/^livecd\/iso/s|'${old_version_stamp}'|'${new_version_stamp}'|' ${i}
  sed -i '/^livecd\/volid/s|'${old_version_stamp}'|'${new_version_stamp}'|' ${i}

  kconfig_lines=$(grep '^boot/kernel/[^/]\+/config:' ${i})
  if [ -n "${kconfig_lines}" ]; then
    echo "${kconfig_lines}" | while read line; do
      key=$(echo "${line}" | cut -d: -f1)
      filename=$(basename $(echo "${line}" | cut -d: -f2))
      sed -i "s|^${key}:.*\$|${key}: ${TMPDIR}/kconfig/${filename}|" ${i}
    done
  fi

  # Expand vars that the spec expects us to.
  sed -i \
    -e "s:@DATESTAMP@:${DATESTAMP}:g" \
    -e "s:@GITDIR@:${GITDIR}:g" \
    -e "s:@REPO_DIR@:${REPO_DIR}:g" \
    "${i}"
done

if [ "${testing}" -eq 1 ]; then
  echo "Exiting due to --test"
  exit
fi

if [[ ${preclean} -eq 1 ]]; then
  snapshot_cache=$(catalyst_var snapshot_cache)
  if [[ -z ${snapshot_cache} ]]; then
    echo "error: snapshot_cache not set in config file"
    exit 1
  fi
  pushd "${BUILD_SRCDIR_BASE}" >/dev/null || exit 1
  rm -rf --one-file-system \
    kerncache packages snapshots tmp "${snapshot_cache}"/*
  popd >/dev/null
fi

# Create snapshot
if ! run_cmd "catalyst -c ${CATALYST_CONFIG} -s ${DATESTAMP}" "${TMPDIR}/log/snapshot.log"; then
  send_email "Catalyst build error - snapshot" "" "${TMPDIR}/log/snapshot.log"
  exit 1
fi

build_failure=0

for a in "" ${SETS}; do
  if [ -z "${a}" ]; then
    specs_var="SPECS"
    optional_specs_var="OPTIONAL_SPECS"
  else
    specs_var="SET_${a}_SPECS"
    optional_specs_var="SET_${a}_OPTIONAL_SPECS"
  fi

  for i in ${!specs_var}; do
    LOGFILE="${TMPDIR}/log/$(echo "${i}" | sed -e 's:/:_:' -e 's:\.spec$::').log"
    run_cmd "time catalyst -a -p -c ${CATALYST_CONFIG} -f ${i}" ${LOGFILE}
    if [ $? != 0 ]; then
      build_failure=1
      send_email "Catalyst fatal build error - ${i}" "" "${LOGFILE}"
      continue 2
    fi
  done

  for i in ${!optional_specs_var}; do
    LOGFILE="${TMPDIR}/log/$(echo "${i}" | sed -e 's:/:_:' -e 's:\.spec$::').log"
    run_cmd "time catalyst -a -p -c ${CATALYST_CONFIG} -f ${i}" ${LOGFILE}
    if [ $? != 0 ]; then
      build_failure=1
      send_email "Catalyst non-fatal build error - ${i}" "" "${LOGFILE}"
      break
    fi
  done

  for i in ${!specs_var} ${!optional_specs_var}; do
    LOGFILE="${TMPDIR}/log/$(echo "${i}" | sed -e 's:/:_:' -e 's:\.spec$::')_purge.log"	
    run_cmd "time catalyst -P -c ${CATALYST_CONFIG} -f ${i}" "${LOGFILE}"
  done

done

if ! run_cmd "post_build" "${TMPDIR}/log/post_build.log"; then
  send_email "Catalyst build error - post_build" "The post_build function failed" "${TMPDIR}/log/post_build.log"
  exit 1
fi

if [ ${build_failure} = 0 ]; then
  if [[ ${lastrun} -ne 0 ]]; then
    stamp=$(date)
    (date -d"${stamp}" +%s; echo "${stamp}") >"${last_success_file}"
  fi

  send_email "Catalyst build success" "Build process complete."

  if [ "${keep_tmpdir}" = 0 ]; then
    if ! rm -rf "${TMPDIR}"; then
      echo "Could not remove tmpdir ${TMPDIR}!"
      exit 1
    fi
  fi

else
  send_email "Catalyst build complete, but with errors" "Build process has completed, but there were errors. Please consult previous emails to determine the problem."
fi

) 9>"${lock_file:-/dev/null}"