sync/ports
1
0
Fork 0
mirror of https://git.freebsd.org/ports.git synced 2025-07-18 17:59:20 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Exit with error status in ${portbuild}/status when an error occurs.

Browse source 
Make INDEX build failure a fatal error.

Copy INDEX to INDEX-${branch} for later perusal.

There is no -a flag in OpenSSH's scp.

Use sed to remove /usr/ports and stuff from restricted.sh and
cdrom.sh, so they can be run without further modification using
relative directories.

Take more options, namely -norestr (don't build RESTRICTED ports),
-noplistcheck (turn off "extra files check"), and -nodummy (use
open-motif instead of Motif-dummy and XFree86-4-* instead of
XFree86-3).  -nodummy will become the default soon.  Also a new flag
-nocvs will prevent cvs update on ${branch}/src only (while -nocvsup
will prevent cvsup and cvs update on usr/ports).  -nocvs is to be used
when builds for more than one branch shares a same cvsup session.

Write out time cvsup is done so it can be used by log processing
scripts.

Handle dummy ports by using a more generic approach -- list the
directories they reside in (e.g., x11-toolkits/Motif-dummy) and
generate the package name from there, instead of hardcoding them
package names everywhere.  Hopefully this will go away when we get rid
of the special case handling for Motif and XFree86.

Add something called "quickports" -- these are packages with very long
dependency lists that always seem to throttle the parallel build near
the end.  By spelling them out explicitly, try to make make build
dependencies for them earlier while there are a lot of other stuff to
do.  (Our make doesn't do anything with the target list, but maybe I
can fix that someday.)

Copy bindist.tar to clients before the second run of package build
too.  It can be useful to "sneak" in fixes if something minorly wrong
is discovered in the bindist during the first run.  Since there is a
md5 checking, this copying will not be done unless the files are
actually different.

When -norestr is set, delete restricted packages and distfiles using
restricted.sh immediately after completion of package build.

Generate CHECKSUM.MD5 after packages are built.

Copy distfiles over to the ftp site after everything is done.  This is
run in the background on bak/distfiles so the next package build can
immediately start.
This commit is contained in:
Satoshi Asami 2000-08-29 08:51:57 +00:00
parent 03914a93bd
commit 31fb70c70d
Notes: svn2git 2021-03-31 03:12:20 +00:00 
svn path=/head/; revision=32091
1 changed files with 170 additions and 45 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

215
Tools/portbuild/scripts/dopackages
View file

@ -7,11 +7,19 @@ user=asami
# packages for dependencies only
dummyports="x11/XFree86 x11-toolkits/Motif-dummy"
date=$(date '+%Y%m%d')
# packages with very long dependency lists -- try to start building these first
quickports="x11/kde11 x11/kde11-i18n x11/gnome"
status=${pb}/status
errorexit () {
echo "$1" > ${status}
exit $1
}
usage () {
echo "usage: [-nobuild] [-noindex] [-noduds] [-nocvsup] branch"
exit 1
echo "usage: [-nobuild] [-noindex] [-noduds] [-nocvsup] [-nocvs] [-norestr] [-noplistcheck] [-nodummy] branch date"
errorexit 1
}
# usage: makeindex pb scripts branch user
@ -26,18 +34,11 @@ makeindex () {
echo "generating index"
echo "================================================"
echo "index generation started at $(date)"
${scripts}/makeindex ${branch} || exit 1
${scripts}/makeindex ${branch} || errorexit 1
echo "index generation ended at $(date)"
if grep -q non-existent INDEX; then
echo "errors in INDEX:"
grep non-existent INDEX
exit 1
# grep -C non-existent INDEX
# grep -v non-existent INDEX > INDEX.tmp
# mv -f INDEX.tmp INDEX
fi
echo $(wc -l <INDEX) "lines in INDEX"
chown ${user} INDEX
cp -p INDEX INDEX-${branch}
}
# usage: checkindex pb
@ -45,9 +46,17 @@ checkindex () {
pb=$1
cd ${pb}/usr/ports
if grep -q non-existent INDEX; then
echo "errors in INDEX:"
grep -n non-existent INDEX
errorexit 1
# grep -C non-existent INDEX
# grep -v non-existent INDEX > INDEX.tmp
# mv -f INDEX.tmp INDEX
fi
if ! awk -F '|' '{if (NF != 10) { error=1; printf("line %d: %s\n", NR, $0)}} END {if (error == 1) exit(1)}' INDEX; then
echo "error in INDEX"
exit 1
errorexit 1
fi
}
@ -64,7 +73,7 @@ makeduds () {
echo "duds generation started at $(date)"
if ! ${scripts}/makeduds ${branch}; then
echo "error(s) detected, exiting script at $(date)"
exit 1
errorexit 1
fi
echo "duds generation ended at $(date)"
echo $(wc -l < ${pb}/${branch}/duds) "items in duds"
@ -83,7 +92,8 @@ setupnode () {
md5=$6
echo "setting up of $node started at $(date)"
scp -p -a ${scripts}/setupnode ${node}:${scripts}
ssh -n ${node} mkdir -p ${scripts}
scp -p ${scripts}/setupnode ${node}:${scripts}
ssh -n ${node} ${scripts}/setupnode ${me} ${pb} ${branch} ${md5}
echo "setting up of $node ended at $(date)"
}
@ -99,7 +109,10 @@ restrictedlist () {
echo "creating restricted list"
echo "================================================"
echo "restricted list generation started at $(date)"
make ECHO_MSG=/usr/bin/true clean-restricted-list > ${pb}/${branch}/restricted.sh
make ECHO_MSG=/usr/bin/true clean-restricted-list \
| sed -e "s./usr/ports/distfiles/./distfiles/.g" \
-e "s./usr/ports/./${branch}/.g" \
> ${pb}/${branch}/restricted.sh
echo "restricted list generation ended at $(date)"
echo $(grep -c '^#' ${pb}/${branch}/restricted.sh) "ports in ${pb}/${branch}/restricted.sh"
}
@ -115,7 +128,10 @@ cdromlist () {
echo "creating cdrom list"
echo "================================================"
echo "cdrom list generation started at $(date)"
make ECHO_MSG=/usr/bin/true clean-for-cdrom-list > ${pb}/${branch}/cdrom.sh
make ECHO_MSG=/usr/bin/true clean-for-cdrom-list \
| sed -e "s./usr/ports/distfiles/./distfiles/.g" \
-e "s./usr/ports/./${branch}/.g" \
> ${pb}/${branch}/cdrom.sh
echo "cdrom list generation ended at $(date)"
echo $(grep -c '^#' ${pb}/${branch}/cdrom.sh) "ports in ${pb}/${branch}/cdrom.sh"
}
@ -142,9 +158,13 @@ nobuild=0
noindex=0
noduds=0
nocvsup=0
nocvs=0
norestr=0
noplistcheck=0
nodummy=0
# optional arguments
while [ $# -gt 1 ]; do
while [ $# -gt 2 ]; do
case "x$1" in
x-nobuild)
nobuild=1
@ -158,6 +178,18 @@ while [ $# -gt 1 ]; do
x-nocvsup)
nocvsup=1
;;
x-nocvs)
nocvs=1
;;
x-norestr)
norestr=1
;;
x-noplistcheck)
noplistcheck=1
;;
x-nodummy)
nodummy=1
;;
*)
usage
;;
@ -165,18 +197,34 @@ while [ $# -gt 1 ]; do
shift
done
# mandatory argument
# mandatory arguments
branch=$1
if [ "x$branch" != x3 -a "x$branch" != x4 ]; then
date=$2
if [ "x$branch" != x3 -a "x$branch" != x4 -a "x$branch" != x5 ]; then
usage
fi
if [ "$norestr" = 1 ]; then
export NO_RESTRICTED=t
fi
if [ "$noplistcheck" = 1 ]; then
export NOPLISTCHECK=t
fi
if [ "$nodummy" = 1 ]; then
export NODUMMY=t
export MOTIF_OPEN=t
export XFREE86_VERSION=4
fi
cd ${pb}/usr/ports
if [ "$nocvsup" = 0 ]; then
echo "================================================"
echo "running cvsup"
echo "================================================"
su ${user} -c 'cvsup -g -L 0 /etc/supfile.cvsup'
date > ${pb}/cvsdone
echo "================================================"
echo "running cvs update on /usr/ports"
echo "================================================"
@ -192,7 +240,15 @@ if [ "$nocvsup" = 0 ]; then
su ${user} -c 'cvs -q update -d -P'
fi
# these two not in background to check return status
if [ "$nocvs" = 0 ]; then
echo "================================================"
echo "running cvs update on /${branch}/src"
echo "================================================"
cd ${pb}/${branch}/src
su ${user} -c 'cvs -q update -d -P'
fi
# this one not in background to check return status
if [ "$noduds" = 0 ]; then
makeduds ${pb} ${scripts} ${branch}
@ -223,38 +279,71 @@ checkindex ${pb}
echo "all preparation ended at $(date)"
cd ${pb}/usr/ports
make parallel > ../../${branch}/Makefile
make parallel branch=${branch} > ../../${branch}/Makefile
cd ${pb}/usr/ports
if [ "$nodummy" = 0 ]; then
dummypkgs=""
for i in ${dummyports}; do
if [ -d $i ]; then
dummypkgs="${dummypkgs} $(cd $i; make package-name).tgz"
else
echo "dummy port directory \"$i\" does not exist -- skipping"
fi
done
fi
quickpkgs=""
for i in ${quickports}; do
if [ -d $i ]; then
quickpkgs="${quickpkgs} $(cd $i; make package-name).tgz"
else
echo "quick port directory \"$i\" does not exist -- skipping"
fi
done
cd ${pb}
if [ "$nobuild" = 0 ]; then
rm -rf bak/distfiles
mv -f distfiles bak
### rm -rf bak/distfiles
### mv -f distfiles bak
mkdir distfiles
chown -R ${user} distfiles
cd ${pb}/${branch}/bak
rm -rf errors logs packages old-errors
cd ${pb}/${branch}
mv -f errors logs packages old-errors make.* bak
cd bak/packages/All
for i in *.tgz; do
cp /dev/null $i
done
cd ${pb}/${branch}
mkdir -p packages/All
mkdir -p ${pb}/archive/errorlogs
rm -rf ${pb}/archive/errorlogs/errors.${branch}.${date}
mkdir -p ${pb}/archive/errorlogs/errors.${branch}.${date}
ln -sf ${pb}/archive/errorlogs/errors.${branch}.${date} ${pb}/${branch}/errors
rm -rf ${pb}/archive/errorlogs/alllogs.${branch}.${date}
mkdir -p ${pb}/archive/errorlogs/alllogs.${branch}.${date}
ln -sf ${pb}/archive/errorlogs/alllogs.${branch}.${date} ${pb}/${branch}/logs
chown -RL ${user} errors logs packages
echo "error logs in ${pb}/archive/errorlogs/errors.${branch}.${date}"
for dir in ${dummyports}; do
cp -p tarballs/$(cd ${pb}/usr/ports/$dir; make package-name).tgz packages/All
done
count=$(awk '{sum+=$2}END{print sum+NR/2}' ${pb}/mlist)
rm -rf ${pb}/archive/errorlogs/e.${branch}.${date}
mkdir -p ${pb}/archive/errorlogs/e.${branch}.${date}
ln -sf ${pb}/archive/errorlogs/e.${branch}.${date} ${pb}/${branch}/errors
rm -rf ${pb}/archive/errorlogs/a.${branch}.${date}
mkdir -p ${pb}/archive/errorlogs/a.${branch}.${date}
ln -sf ${pb}/archive/errorlogs/a.${branch}.${date} ${pb}/${branch}/logs
cp -p ${pb}/cvsdone ${pb}/archive/errorlogs/e.${branch}.${date}/cvsdone
cp -p ${pb}/cvsdone ${pb}/archive/errorlogs/a.${branch}.${date}/cvsdone
cp -p ${pb}/usr/ports/INDEX-${branch} ${pb}/archive/errorlogs/e.${branch}.${date}/INDEX
cp -p ${pb}/usr/ports/INDEX-${branch} ${pb}/archive/errorlogs/a.${branch}.${date}/INDEX
chown -RL ${user} errors logs packages
echo "error logs in ${pb}/archive/errorlogs/e.${branch}.${date}"
if [ "$nodummy" = 0 ]; then
for dir in ${dummyports}; do
cp -p tarballs/$(cd ${pb}/usr/ports/$dir; make package-name).tgz packages/All
done
fi
count=$(awk '{sum+=$2}END{print sum+NR/2}' ${pb}/mlist | sed -e 's/\..*$//')
cd ${pb}/${branch}/packages/All
echo "================================================"
echo "building packages (phase 1)"
echo "================================================"
echo "started at $(date)"
make -k -j$count -f ../../Makefile > ../../make.0 2>&1 </dev/null
make -k -j$count -f ../../Makefile ${quickpkgs} all > ../../make.0 2>&1 </dev/null
echo "ended at $(date)"
echo $(echo $(ls -1 ${pb}/${branch}/packages/All | wc -l) - 2 | bc) "packages built"
echo $(echo $(du -sk ${pb}/${branch}/packages | awk '{print $1}') / 1024 | bc) "MB of packages"
@ -267,21 +356,48 @@ if [ "$nobuild" = 0 ]; then
cp -rp errors old-errors
cd ${pb}/${branch}/old-errors
${pb}/scripts/processlogs
md5=$(/sbin/md5 ${pb}/${branch}/tarballs/bindist.tar | awk '{print $4}')
echo "================================================"
echo "setting up nodes"
echo "================================================"
for node in $(awk '{print $1}' ${pb}/mlist); do
setupnode ${pb} ${scripts} ${branch} ${me} ${node} ${md5} &
sleep 2
done
wait
echo "setting up of nodes ended at $(date)"
cd ${pb}/${branch}/packages/All
echo "================================================"
echo "building packages (phase 2)"
echo "================================================"
echo "started at $(date)"
make -k -j$count -f ../../Makefile > ../../make.1 2>&1 </dev/null
make -k -j$count -f ../../Makefile ${quickpkgs} all > ../../make.1 2>&1 </dev/null
echo "ended at $(date)"
rm XFree86-3.*.tgz Motif-*.tgz
${pb}/scripts/chopindex ${pb}/usr/ports/INDEX ${pb}/${branch}/packages > \
if [ "$nodummy" = 0 ]; then
rm ${dummypkgs}
fi
${pb}/scripts/chopindex ${pb}/usr/ports/INDEX-${branch} ${pb}/${branch}/packages > \
${pb}/${branch}/packages/INDEX
echo $(ls -1 ${pb}/${branch}/packages/All | wc -l) "packages built"
echo $(cat ${pb}/${branch}/packages/INDEX | wc -l) "lines in INDEX"
md5 *.tgz > CHECKSUM.MD5
echo $(echo $(du -sk ${pb}/${branch}/packages | awk '{print $1}') / 1024 | bc) "MB of packages"
echo $(echo $(du -sk ${pb}/distfiles | awk '{print $1}') / 1024 | bc) "MB of distfiles"
if [ "$norestr" = 0 ]; then
echo "deleting restricted ports"
sh ${pb}/${branch}/restricted.sh
${pb}/scripts/chopindex ${pb}/usr/ports/INDEX ${pb}/${branch}/packages > \
${pb}/${branch}/packages/INDEX
echo $(ls -1 ${pb}/${branch}/packages/All | wc -l) "packages built"
echo $(cat ${pb}/${branch}/packages/INDEX | wc -l) "lines in INDEX"
echo $(echo $(du -sk ${pb}/${branch}/packages | awk '{print $1}') / 1024 | bc) "MB of packages"
echo $(echo $(du -sk ${pb}/distfiles | awk '{print $1}') / 1024 | bc) "MB of distfiles"
fi
echo "generating CHECKSUM.MD5"
md5 *.tgz > CHECKSUM.MD5
cd ${pb}/${branch}
if grep -q 'ptimeout: killing' make.1; then
echo "The following port(s) timed out:"
@ -302,7 +418,7 @@ if [ "$nobuild" = 0 ]; then
echo "================================================"
cd ${pb}/${branch}/errors
for i in *.log; do
if [ ! -f ../bak/errors/$i ]; then
if [ ! -f ${pb}/${branch}/bak/errors/$i ]; then
echo -n " $(basename $i .log)"
fi
done
@ -312,7 +428,7 @@ if [ "$nobuild" = 0 ]; then
echo "================================================"
cd ${pb}/${branch}/bak/packages/All
for i in *.tgz; do
if [ ! -f ../../../packages/All/$i ]; then
if [ ! -f ${pb}/${branch}/packages/All/$i ]; then
echo -n " $(basename $i .tgz)"
fi
done
@ -322,7 +438,7 @@ if [ "$nobuild" = 0 ]; then
echo "================================================"
cd ${pb}/${branch}/bak/errors
for i in *.log; do
if [ ! -f ../../errors/$i ]; then
if [ ! -f ${pb}/${branch}/errors/$i ]; then
echo -n " $(basename $i .log)"
fi
done
@ -332,7 +448,7 @@ if [ "$nobuild" = 0 ]; then
echo "================================================"
cd ${pb}/${branch}/packages/All
for i in *.tgz; do
if [ ! -f ../../bak/packages/All/$i ]; then
if [ ! -f ${pb}/${branch}/bak/packages/All/$i ]; then
echo -n " $(basename $i .tgz)"
fi
done
@ -343,6 +459,15 @@ if [ "$nobuild" = 0 ]; then
echo "started at $(date)"
cd ${pb}/usr
tar --exclude CVS -czf ${pb}/${branch}/tarballs/ports.tar.gz ports
echo "ended at $(date)"
echo "================================================"
echo "copying distfiles"
echo "================================================"
echo "started at $(date)"
cd ${pb}
rm -rf bak/distfiles
mv -f distfiles bak
su ${user} -c ${scripts}/cpdistfiles > ${pb}/cpdistfiles.log 2>&1 </dev/null &
fi
echo "================================================"
echo "all done at $(date)"