chadmed/gentoo-asahi-releng
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

scripts/copy_buildsync: dead detection works too well, send to files instead

Browse source 
Signed-off-by: Robin H. Johnson <robbat2@gentoo.org>
This commit is contained in:
Robin H. Johnson 2021-11-12 09:59:08 -08:00
parent ebb201a6dd
commit 8623ef538c
No known key found for this signature in database
GPG key ID: 19395F23C58826C4
1 changed files with 12 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
scripts/copy_buildsync.sh
View file

@ -6,6 +6,8 @@ INCOMING_BASE="/release/weekly/builds"
OUTGOING_BASE="/release/distfiles/weekly" OUTGOING_BASE="/release/distfiles/weekly"
# Scratch space used when moving files from incoming to outgoing. # Scratch space used when moving files from incoming to outgoing.
TMPDIR_BASE="/release/distfiles/tmp/buildsync/partial" TMPDIR_BASE="/release/distfiles/tmp/buildsync/partial"
# Keep some records
LOGDIR_BASE="/release/distfiles/tmp/buildsync/logs"
ARCHES=( ARCHES=(
alpha alpha
@ -64,7 +66,7 @@ EOF
# Copy artifacts for an arch to the outgoing directory. # Copy artifacts for an arch to the outgoing directory.
copy_arch_to_outgoing() { copy_arch_to_outgoing() {
local ARCH=$1 indir=$2 outdir=$3 tmpdir=$4 local ARCH=$1 indir=$2 outdir=$3 tmpdir=$4 logdir=$5
local i t rc timestamps local i t rc timestamps
if [[ ! -d ${indir} ]]; then if [[ ! -d ${indir} ]]; then
@ -91,6 +93,7 @@ copy_arch_to_outgoing() {
"${RSYNC_OPTS[@]}" \ "${RSYNC_OPTS[@]}" \
--temp-dir="${tmpdir}" \ --temp-dir="${tmpdir}" \
--partial-dir="${tmpdir}" \ --partial-dir="${tmpdir}" \
--log-file="${logdir}/rsync.log" \
--filter '- **/.*' \ --filter '- **/.*' \
--filter "S *${i}*" \ --filter "S *${i}*" \
--filter 'S **/' \ --filter 'S **/' \
@ -126,11 +129,12 @@ process_arch() {
indir="${INCOMING_BASE}/${ARCH}" indir="${INCOMING_BASE}/${ARCH}"
outdir="${OUTGOING_BASE}/${ARCH}" outdir="${OUTGOING_BASE}/${ARCH}"
tmpdir="${TMPDIR_BASE}/${ARCH}" tmpdir="${TMPDIR_BASE}/${ARCH}"
logdir="${LOGDIR_BASE}/${ARCH}"
mkdir -p "${tmpdir}" 2>/dev/null mkdir -p "${tmpdir}" "${logdir}" 2>/dev/null
# Sync incoming->outgoing first. # Sync incoming->outgoing first.
copy_arch_to_outgoing "${ARCH}" "${indir}" "${outdir}" "${tmpdir}" copy_arch_to_outgoing "${ARCH}" "${indir}" "${outdir}" "${tmpdir}" "${logdir}"
# ================================================================ # ================================================================
# Build data for revealing latest: # Build data for revealing latest:
@ -216,10 +220,7 @@ process_arch() {
# Find the dead links for cleanup # Find the dead links for cleanup
_dead="${tmpdir}"/dead-link _dead="${tmpdir}"/dead-link
find -L $(pwd) -type l >"${_dead}" find -L $(pwd) -type l >"${_dead}"
if test -s "${_dead}"; then mv -f "${_dead}" "${logdir}/dead-links.txt"
echo "copy_buildsync: dead links to verify:" 1>&2
cat "${_dead}" 1>&2
fi
# Find the dead latest txt files # Find the dead latest txt files
_dead="${tmpdir}"/dead-latest _dead="${tmpdir}"/dead-latest
@ -235,6 +236,7 @@ process_arch() {
| fgrep -l -f - $f \ | fgrep -l -f - $f \
| xargs -n1 --no-run-if-empty readlink -f | xargs -n1 --no-run-if-empty readlink -f
done >"${_dead}" done >"${_dead}"
if test -s "${_dead}"; then if test -s "${_dead}"; then
echo "copy_buildsync: removing dead latest*txt files:" 1>&2 echo "copy_buildsync: removing dead latest*txt files:" 1>&2
foreach txtfil in $(cat "${_dead}") ; do foreach txtfil in $(cat "${_dead}") ; do
@ -242,11 +244,14 @@ process_arch() {
done done
fi fi
mv -f "${_dead}" "${logdir}/dead-latest-txt.txt"
# Cleanup tmpdir # Cleanup tmpdir
$DEBUGP rm $VERBOSEP -rf "${tmpdir}" $DEBUGP rm $VERBOSEP -rf "${tmpdir}"
else else
echo "There was some failure for $ARCH during the weekly sync. Not doing cleanup for fear of dataloss." 1>&2 echo "There was some failure for $ARCH during the weekly sync. Not doing cleanup for fear of dataloss." 1>&2
echo "See logs in $logdir" 1>&2
fi fi
} }