2012-07-09 17:47:42 +00:00
#!/bin/bash
2016-03-20 21:44:59 +00:00
# Where artifacts are uploaded by builders.
INCOMING_BASE = "/release/weekly/builds"
# Where artifacts are moved to so they can be uploaded to mirrors.
OUTGOING_BASE = "/release/distfiles/weekly"
# Scratch space used when moving files from incoming to outgoing.
TMPDIR_BASE = "/release/distfiles/tmp/buildsync/partial"
2016-03-20 20:46:11 +00:00
ARCHES = (
alpha
amd64
arm
hppa
ia64
#mips
ppc
s390
sh
sparc
x86
)
RSYNC_OPTS = (
-aO
--delay-updates
)
2016-03-20 23:37:44 +00:00
# Command line for `find` to figure out what files are release artifacts.
EXTENSIONS = (
'('
-name '*.tar.xz' -o
-name '*.tar.bz2' -o
-name '*.tar.gz' -o
-name '*.tar' -o
-name '*.sfs'
')'
)
2012-07-09 17:47:42 +00:00
OUT_STAGE3 = "latest-stage3.txt"
OUT_ISO = "latest-iso.txt"
# Nothing to edit beyond this point
DEBUGP =
VERBOSEP =
2015-10-22 09:59:09 +00:00
2016-03-20 21:10:47 +00:00
usage( ) {
cat <<EOF
2016-03-20 21:18:25 +00:00
Usage: $0 [ options]
2016-03-20 21:10:47 +00:00
Move releases from the incoming upload directory to the outgoing release
directory so they can be pushed out to mirrors.
Also update the "latest" links/files so people can easily find the current
version for any particular arch/release.
2016-03-20 21:18:25 +00:00
Options:
-v, --verbose Run in verbose mode
-d, --debug Run in debug mode
2016-03-20 21:10:47 +00:00
EOF
2016-03-20 21:18:25 +00:00
exit ${ 1 :- 1 }
2016-03-20 21:10:47 +00:00
}
2016-03-20 21:44:59 +00:00
# Copy artifacts for an arch to the outgoing directory.
copy_arch_to_outgoing( ) {
local ARCH = $1 indir = $2 outdir = $3 tmpdir = $4
local i t rc
if [ [ ! -d ${ indir } ] ] ; then
# Nothing to do for this arch.
return
fi
# Copying
2018-11-21 06:43:38 +00:00
for i in $( find ${ indir } -not -path '*/\.*' -type f | egrep -- '-20[0123][0-9]{5}(([0-9]{6})|(T[0-9]{6}Z))?' | sed -e 's:^.*-\(20[^.]\+\).*$:\1:' | sort -ur) ; do
2016-03-20 21:44:59 +00:00
#echo "Doing $i"
t = " ${ outdir } / ${ i } "
mkdir -p ${ t } 2>/dev/null
2018-11-21 06:43:38 +00:00
rsync " ${ RSYNC_OPTS [@] } " --temp-dir= ${ tmpdir } --partial-dir= ${ tmpdir } ${ indir } / --filter '- **/.*' --filter " S * ${ i } * " --filter 'S **/' --filter 'H *' ${ t }
2016-03-20 21:44:59 +00:00
rc = $?
if [ $rc -eq 0 ] ; then
2018-11-21 06:43:38 +00:00
find ${ indir } -not -path '*/\.*' -type f -name " * ${ i } * " -print0 | xargs -0 --no-run-if-empty $DEBUGP rm $VERBOSEP -f
2016-03-20 21:44:59 +00:00
else
echo " Not deleting ${ indir } /* ${ i } *, rsync failed! " 1>& 2
fail = 1
fi
done
2016-03-20 21:55:08 +00:00
find " ${ outdir } " \
-depth -mindepth 1 -type d \
-exec rmdir --ignore-fail-on-non-empty { } +
2016-03-20 21:44:59 +00:00
}
2016-03-20 21:10:47 +00:00
process_arch( ) {
local ARCH = $1
2012-07-09 17:47:42 +00:00
fail = 0
2016-03-20 21:44:59 +00:00
indir = " ${ INCOMING_BASE } / ${ ARCH } "
outdir = " ${ OUTGOING_BASE } / ${ ARCH } "
tmpdir = " ${ TMPDIR_BASE } / ${ ARCH } "
2012-07-09 17:47:42 +00:00
mkdir -p ${ tmpdir } 2>/dev/null
2015-10-22 09:59:09 +00:00
2016-03-20 21:44:59 +00:00
# Sync incoming->outgoing first.
copy_arch_to_outgoing " ${ ARCH } " " ${ indir } " " ${ outdir } " " ${ tmpdir } "
2012-07-09 17:47:42 +00:00
# ================================================================
# Build data for revealing latest:
# *.iso
# stage3*bz2
cd " ${ outdir } "
# %T@
2016-05-06 18:31:55 +00:00
iso_list = " $( find 20* -name '*.iso' -printf '%h %f %h/%f\n' 2>/dev/null | grep -v hardened | sort -n) "
stage3_list = $( find 20* -name "stage3*" -a " ${ EXTENSIONS [@] } " -printf '%h %f %h/%f\n' 2>/dev/null| grep -v hardened | sort -n)
2012-07-09 17:47:42 +00:00
latest_iso_date = " $( echo -e " ${ iso_list } " | awk '{print $1}' | cut -d/ -f1 | tail -n1) "
latest_stage3_date = " $( echo -e " ${ stage3_list } " | awk '{print $1}' | cut -d/ -f1 | tail -n1) "
header = " $( echo -e " # Latest as of $( date -uR) \n# ts= $( date -u +%s) " ) "
2013-04-18 08:04:50 +00:00
2012-07-09 17:47:42 +00:00
# Do not remove this
[ -z " ${ latest_iso_date } " ] && latest_iso_date = "NONE-FOUND"
[ -z " ${ latest_stage3_date } " ] && latest_stage3_date = "NONE-FOUND"
2013-04-18 08:04:50 +00:00
2012-07-09 17:47:42 +00:00
if [ -n " ${ iso_list } " ] ; then
echo -e " ${ header } " >" ${ OUT_ISO } "
2016-03-20 22:29:45 +00:00
# Some arches produce more than one type of iso.
# Only apply the current-iso link logic to them.
# TODO: Should make this dynamic based on the iso list.
case ${ ARCH } in
amd64| x86)
2015-02-06 20:11:54 +00:00
rm -f current-iso
2016-03-20 22:29:45 +00:00
; ;
*)
echo -e " ${ iso_list } " | awk '{print $3}' | grep " $latest_iso_date " >>${ OUT_ISO }
ln -sfT " $latest_iso_date " current-iso
; ;
esac
2012-07-09 17:47:42 +00:00
fi
if [ -n " ${ stage3_list } " ] ; then
echo -e " ${ header } " >" ${ OUT_STAGE3 } "
2015-10-22 09:59:09 +00:00
2013-08-17 18:58:44 +00:00
# In the new variant preserve code there is a better way to do this
#echo -e "${stage3_list}" |awk '{print $3}' |grep "$latest_stage3_date" >>${OUT_STAGE3}
2015-10-22 09:59:09 +00:00
2016-03-20 22:12:57 +00:00
# The "latest stage3" concept works for only a few arches -- ones that
# do not have more than one stage3 target per arch (i.e. multilib, etc...).
case ${ ARCH } in
amd64| arm| hppa| ppc| s390| sh| x86)
rm -f current-stage3
; ;
*)
ln -sfT " $latest_stage3_date " current-stage3
; ;
esac
2012-07-09 17:47:42 +00:00
fi
2013-04-18 08:04:50 +00:00
2015-10-22 09:59:09 +00:00
# New variant preserve code
2016-03-20 23:51:56 +00:00
find_variants = ( '(' -iname '*.iso' -o -name 'netboot-*' -o " ${ EXTENSIONS [@] } " ')' )
2017-12-08 07:40:09 +00:00
variants = $( find 20* " ${ find_variants [@] } " -printf '%f\n' 2>/dev/null | sed -e 's,-20[012][0-9]\{5\}.*,,g' -r | sort -u)
2012-07-09 17:47:42 +00:00
echo -n '' >" ${ tmpdir } " /.keep.${ ARCH } .txt
2013-04-18 08:04:50 +00:00
for v in $variants ; do
2016-05-06 18:31:55 +00:00
variant_path = $( find 20* -iname " ${ v } -20* " " ${ find_variants [@] } " -print 2>/dev/null | sed -e " s,.*/ $a /autobuilds/,,g " | sort -k1,1 -t/ | tail -n1 )
2015-10-20 17:30:13 +00:00
if [ -z " ${ variant_path } " -o ! -e " ${ variant_path } " ] ; then
echo " $ARCH : Variant ${ v } is missing " 1>& 2
continue
fi
2015-02-06 20:11:54 +00:00
size = $( stat --format= %s ${ variant_path } )
2012-07-09 17:47:42 +00:00
f = " latest- ${ v } .txt "
echo -e " ${ header } " >" ${ f } "
2015-02-06 20:11:54 +00:00
echo -e " ${ variant_path } ${ size } " >>${ f }
[ [ ${ variant_path } = ~ tar.*$ ] ] && echo -e " ${ variant_path } ${ size } " >>${ OUT_STAGE3 }
[ [ ${ variant_path } = ~ iso$ ] ] && echo -e " ${ variant_path } ${ size } " >>${ OUT_ISO }
2012-07-09 17:47:42 +00:00
rm -f " current- $v "
ln -sf " ${ variant_path %/* } " " current- $v "
echo " ${ variant_path } " | sed -e 's,/.*,,g' -e 's,^,/,g' -e 's,$,$,g' >>" ${ tmpdir } " /.keep.${ ARCH } .txt
2013-04-18 08:04:50 +00:00
done
2012-07-09 17:47:42 +00:00
# ================================================================
# Cleanup
if [ $fail -eq 0 ] ; then
2015-10-22 09:59:09 +00:00
2012-07-09 17:47:42 +00:00
# Clean up all but latest 4 from mirror dir
cd " ${ outdir } "
for i in $( find -regextype posix-basic -mindepth 1 -maxdepth 1 -type d -regex '.*20[012][0-9]\{5\}.*' \
| sed -e 's:^.*-\(20[^.]\+\).*$:\1:' \
| sort -ur \
| egrep -v " / ${ latest_iso_date } \$|/ ${ latest_stage3_date } \$ " \
| egrep -v -f " ${ tmpdir } " /.keep.${ ARCH } .txt \
| tail -n +5) ; do
$DEBUGP rm $VERBOSEP -rf $( pwd ) /${ i }
done
2013-04-18 08:04:50 +00:00
2012-07-09 17:47:42 +00:00
$DEBUGP rm $VERBOSEP -rf ${ tmpdir }
else
echo " There was some failure for $ARCH during the weekly sync. Not doing cleanup for fear of dataloss. " 1>& 2
fi
2016-03-20 21:10:47 +00:00
}
2012-07-09 17:47:42 +00:00
2016-03-20 21:10:47 +00:00
main( ) {
2016-03-20 21:18:25 +00:00
# Process all the command line options first.
while [ [ $# -ne 0 ] ] ; do
case $1 in
-d| --debug)
DEBUGP = "echo"
RSYNC_OPTS += ( -n )
; ;
-v| --verbose)
VERBOSEP = "-v"
RSYNC_OPTS += ( -v )
; ;
-h| --help)
usage 0
; ;
*)
usage 1
; ;
esac
shift
done
2016-03-20 21:10:47 +00:00
2016-03-20 21:18:25 +00:00
# Process all the architectures.
2016-03-20 21:10:47 +00:00
local arch
for arch in " ${ ARCHES [@] } " ; do
process_arch " ${ arch } "
done
}
main " $@ "
2012-07-09 17:47:42 +00:00
# vim:ts=2 sw=2 noet ft=sh: