sync/ports
1
0
Fork 0
mirror of https://git.freebsd.org/ports.git synced 2025-07-18 01:39:16 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Rewrite of processlog by kris.

Browse source
This commit is contained in:
Mark Linimon 2009-03-26 10:33:51 +00:00
parent c65d99ec8b
commit 4b88f15929
Notes: svn2git 2021-03-31 03:12:20 +00:00 
svn path=/head/; revision=231054
1 changed files with 148 additions and 85 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

227
Tools/portbuild/scripts/processlogs
View file

@ -1,4 +1,19 @@
#!/bin/sh #!/bin/sh
# Process the logs in a certain directory and construct the HTML pages
# summarizing them
#
# We only construct the new html pages if either
# * No HTML files exist yet
# * .force exists in this directory (set by pdispatch when a log is
# removed by package build success)
# * A new log file has appeared since the last run.
#
# Individual summary data for the logs is saved between runs instead of
# being recomputed each time. Removing the old data for logs that
# no longer exist is the responsibility of the dopackages script when
# starting a new build.
# echo 'processlogs: at '`date`', begin' # echo 'processlogs: at '`date`', begin'
arch=$1 arch=$1
@ -6,14 +21,18 @@ arch=$1
# establish which directory the logfiles live in (leave out for # establish which directory the logfiles live in (leave out for
# backwards compatibility) # backwards compatibility)
buildlogdir=$(realpath .) buildlogdir=$(realpath .)
if [ "$2" != "" ]; then buildlogdir="$2"; fi if [ "$2" != "" ]; then
buildlogdir="$2"
fi
baselogdir=$(basename ${buildlogdir}) baselogdir=$(basename ${buildlogdir})
# establish which directory INDEX lives in (leave out for # establish which directory INDEX lives in (leave out for
# backwards compatibility) # backwards compatibility)
indexlogdir=$(realpath .) indexlogdir=$(realpath .)
if [ "$3" != "" ]; then indexlogdir="$3"; fi if [ "$3" != "" ]; then
indexlogdir="$3";
fi
# allow this script to be run from anywhere in the tree # allow this script to be run from anywhere in the tree
scriptdir=$(dirname $0) scriptdir=$(dirname $0)
@ -24,18 +43,29 @@ pb=/var/portbuild
. ${pb}/${arch}/portbuild.conf . ${pb}/${arch}/portbuild.conf
# get the list of buildlogs.
cd ${buildlogdir}
if [ -e .force -o \! -e index.html ]; then
rm -f .force
force=1
else
force=0
fi
alllogs=$(find . -maxdepth 1 -type f \( -name '*.log' -o -name '*.log.bz2' \) | sed -e 's,^./,,' 2>/dev/null)
if [ -e .stamp -a ${#alllogs} -eq 0 ]; then
exit
fi
# create a name for the tempfile # create a name for the tempfile
of=.index.html of=.index.html
# if there are no new logfiles, there is nothing to do here. # XXX temporary
if [ \! -e .force -a -e .stamp -a $(echo $(find -f $buildlogdir -maxdepth 1 -newer .stamp -type f \( -name '*.log' -o -name '*.log.bz2' \) 2>/dev/null | wc -l)) = "0" ]; then exit; fi test -f .logs && rm -f .logs
mkdir -p .logs
rm -f .force
touch .stamp
# get the list of buildlogs.
set $buildlogdir/*.log $buildlogdir/*.log.bz2
> .logs
# #
# Read the log-files and write summaries to .logs in the format # Read the log-files and write summaries to .logs in the format
@ -43,24 +73,45 @@ set $buildlogdir/*.log $buildlogdir/*.log.bz2
# $reason|$tag|$broken|$datetime # $reason|$tag|$broken|$datetime
# #
# echo 'processlogs: at '`date`', begin processing log files' echo "processlogs: at $(date), begin processing log files for ${arch} in ${baselogdir}"
if [ $# != 1 -o "x$1" != "x`dirname $1`/*.log" -o "x$1" != "x`dirname $1`/*.log.bz2" ]; then rm -f .logs/.all
while [ ! -z "$1" ]; do set -- ${alllogs}
filename=$(basename $1)
#echo 'processlogs: at '`date`', examining '$filename'
$errorscript $filename $indexlogdir >> .logs gotnew=0
while [ $# -ge 1 ]; do
log=$1
shift shift
done
if [ ${log%.log.bz2} != ${log} -a -e ${log%.bz2} ]; then
# We have both a .log.bz2 and a .log, assume the former is stale
rm -f ${log}
continue
fi fi
# echo 'processlogs: at '`date`', end processing log files'
# XXX Sometimes log entries get doubled up for some reason # basename with .log and .log.bz2 stripped
uniq .logs > .logs2 base=${log%.bz2}
mv .logs2 .logs base=${log%.log}
num=$(wc -l < .logs) if [ ${log} -nt .stamp -o ! -e .logs/${base} ]; then
# Save to the base name so the data persists after the log is
# compressed
${errorscript} ${log} ${indexlogdir} > .logs/${base}
gotnew=1
fi
cat .logs/${base} >> .logs/.all
done
touch .stamp
echo "processlogs: at "`date`", end processing log files for ${arch} in ${baselogdir}"
if [ ${force} -eq 0 -a ${gotnew} -eq 0 ]; then
# Nothing new to do
exit
fi
header() { header() {
echo "<html><head><title>Package building errors</title>" >$of echo "<html><head><title>Package building errors</title>" >$of
@ -73,17 +124,17 @@ header() {
echo "| <a href=\"index-builddate.html\">builddate</a> " >>$of echo "| <a href=\"index-builddate.html\">builddate</a> " >>$of
echo "]</p>" >>$of echo "]</p>" >>$of
if [ $num = "0" ]; then if [ ${#alllogs} = "0" ]; then
echo "No errors (yet)" >>$of echo "No errors (yet)" >>$of
else else
if [ -s cvsdone ]; then if [ -s cvsdone ]; then
echo "CVS update finished at: $(cat cvsdone)<br>" >> $of echo "CVS update finished at: $(cat cvsdone)<br>" >> $of
fi fi
latest=$(cd $buildlogdir; ls -rtTl *.log *.log.bz2 | tail -1 | awk '{printf("%s %s %s %s\n",$6,$7,$8,$9)}';) latest=$(ls -rtTl *.log *.log.bz2 2> /dev/null | tail -1 | awk '{printf("%s %s %s %s\n",$6,$7,$8,$9)}')
echo "Timestamp of newest log: $latest<br><br>" >> $of echo "Timestamp of newest log: $latest<br><br>" >> $of
echo "\"Aff.\" is number of ports that depend on this one<br>" >> $of echo "\"Aff.\" is number of ports that depend on this one<br>" >> $of
echo "\"<font color=\"red\">[B]</font>\" indicates port is marked BROKEN (Note: BROKEN ports are not frequently rebuilt so they may not be listed here)<br><br>" >> $of echo "\"<font color=\"red\">[B]</font>\" indicates port is marked BROKEN (Note: BROKEN ports are not frequently rebuilt so they may not be listed here)<br><br>" >> $of
echo "<p>$num errors</p>" >> $of echo "<p>${#alllogs} errors</p>" >> $of
echo "<table border=1>" >>$of echo "<table border=1>" >>$of
echo "<tr>$1</tr>" >>$of echo "<tr>$1</tr>" >>$of
fi fi
@ -95,10 +146,7 @@ footer() {
echo "</html>" >>$of echo "</html>" >>$of
} }
# Now reread the .logs file and create the reports. If the .logs file # Now reread the .logs/.all file and create the reports.
# has blanks in any field, this code won't work. Therefore, the
# code above has to guarantee that that won't happen, and the code
# below has to catch the metatoken.
# echo 'processlogs: at '`date`', create default output' # echo 'processlogs: at '`date`', create default output'
# #
@ -106,10 +154,13 @@ footer() {
# #
header "<th>Port</th><th>Aff.</th><th>Size</th><th>CVS</th><th>Maintainer</th><th>Reason</th><th>Build date</th>" header "<th>Port</th><th>Aff.</th><th>Size</th><th>CVS</th><th>Maintainer</th><th>Reason</th><th>Build date</th>"
for i in `cat .logs | sort`; do if [ ${#alllogs} -gt 0 ]; then
set $(echo "$i" | tr \| " " | sed -e "s@NONE@\\&nbsp;@g") sort .logs/.all | while read line; do
IFS="|"
set -- ${line}
unset IFS
mailto="$6" mailto="$6"
if [ "$6" != "&nbsp;" ] ; then if [ "$6" != "" ] ; then
mailto="<a href=\"mailto:$6\">$6</a>" mailto="<a href=\"mailto:$6\">$6</a>"
fi fi
@ -117,23 +168,24 @@ for i in `cat .logs | sort`; do
echo "<td><a href=\"http://${master}/errorlogs/${arch}-errorlogs/${baselogdir}/$1\">$2</a></td>" >> $of echo "<td><a href=\"http://${master}/errorlogs/${arch}-errorlogs/${baselogdir}/$1\">$2</a></td>" >> $of
affby=$3 affby="$3"
test $affby = "0" -o $affby = "-1" && affby="&nbsp;" test "${affby}" = "0" -o "${affby}" = "-1" && affby="&nbsp;"
echo "<td align=\"right\">$affby</td><td align=\"right\">$4 Kb</td>" >> $of echo "<td align=\"right\">${affby}</td><td align=\"right\">$4 Kb</td>" >> $of
echo "<td><a href=\"http://www.FreeBSD.org/cgi/cvsweb.cgi/ports/$5\">$5</a></td>" >> $of echo "<td><a href=\"http://www.FreeBSD.org/cgi/cvsweb.cgi/ports/$5\">$5</a></td>" >> $of
echo "<td>$mailto</td>" >> $of echo "<td>$mailto</td>" >> $of
echo "<td>" >> $of echo "<td>" >> $of
test "$9" = "broken" && echo "<font color=\"red\">[B]</font>" >> $of test "$9" = "yes" && echo "<font color=\"red\">[B]</font>" >> $of
reason=$(echo $7 | tr '_' ' ') reason="$7"
echo "<a href=\"http://pointyhat.freebsd.org/#$8\">$reason</a>" >> $of echo "<a href=\"http://pointyhat.freebsd.org/#$8\">$reason</a>" >> $of
echo "</td>" >> $of echo "</td>" >> $of
date=$(echo ${10} | tr '_' ' ') date="${10}"
echo "<td>$date</td>" >> $of echo "<td>$date</td>" >> $of
echo "</tr>" >> $of echo "</tr>" >> $of
done done
fi
footer "" footer ""
mv -f $of index.html mv -f $of index.html
@ -143,10 +195,13 @@ mv -f $of index.html
# #
header "<th>CVS</th><th>Aff.</th><th>Size</th><th>Port</th><th>Maintainer</th><th>Reason</th><th>Build date</th>" header "<th>CVS</th><th>Aff.</th><th>Size</th><th>Port</th><th>Maintainer</th><th>Reason</th><th>Build date</th>"
for i in `cat .logs | sort -t \\| +4`; do if [ ${#alllogs} -gt 0 ]; then
set $(echo "$i" | tr \| " " | sed -e "s@NONE@\\&nbsp;@g") sort -t \| +4 .logs/.all | while read line; do
IFS="|"
set -- $line
unset IFS
mailto="$6" mailto="$6"
if [ "$6" != "&nbsp;" ] ; then if [ "$6" != "" ] ; then
mailto="<a href=\"mailto:$6\">$6</a>" mailto="<a href=\"mailto:$6\">$6</a>"
fi fi
@ -154,23 +209,24 @@ for i in `cat .logs | sort -t \\| +4`; do
echo "<td><a href=\"http://www.FreeBSD.org/cgi/cvsweb.cgi/ports/$5\">$5</a></td>" >> $of echo "<td><a href=\"http://www.FreeBSD.org/cgi/cvsweb.cgi/ports/$5\">$5</a></td>" >> $of
affby=$3 affby="$3"
test $affby = "0" -o $affby = "-1" && affby="&nbsp;" test "${affby}" = "0" -o "${affby}" = "-1" && affby="&nbsp;"
echo "<td align=\"right\">$affby</td><td align=\"right\">$4 Kb</td>" >> $of echo "<td align=\"right\">${affby}</td><td align=\"right\">$4 Kb</td>" >> $of
echo "<td><a href=\"http://${master}/errorlogs/${arch}-errorlogs/${baselogdir}/$1\">$2</a></td>" >> $of echo "<td><a href=\"http://${master}/errorlogs/${arch}-errorlogs/${baselogdir}/$1\">$2</a></td>" >> $of
echo "<td>$mailto</td>" >> $of echo "<td>$mailto</td>" >> $of
echo "<td>" >> $of echo "<td>" >> $of
test "$9" = "broken" && echo "<font color=\"red\">[B]</font>" >> $of test "$9" = "broken" && echo "<font color=\"red\">[B]</font>" >> $of
reason=$(echo $7 | tr '_' ' ') reason="$7"
echo "<a href=\"http://pointyhat.freebsd.org/#$8\">$reason</a>" >> $of echo "<a href=\"http://pointyhat.freebsd.org/#$8\">$reason</a>" >> $of
echo "</td>" >> $of echo "</td>" >> $of
date=$(echo ${10} | tr '_' ' ') date="${10}"
echo "<td>$date</td>" >> $of echo "<td>$date</td>" >> $of
echo "</tr>" >> $of echo "</tr>" >> $of
done done
fi
footer "" footer ""
mv -f $of index-category.html mv -f $of index-category.html
@ -180,10 +236,13 @@ mv -f $of index-category.html
# #
header "<th>Maintainer</th><th>Port</th><th>Aff.</th><th>Size</th><th>CVS</th><th>Reason</th><th>Build date</th>" header "<th>Maintainer</th><th>Port</th><th>Aff.</th><th>Size</th><th>CVS</th><th>Reason</th><th>Build date</th>"
for i in `cat .logs | sort -t \\| +5`; do if [ ${#alllogs} -gt 0 ]; then
set $(echo "$i" | tr \| " " | sed -e "s@NONE@\\&nbsp;@g") sort -t \| +5 .logs/.all | while read line; do
IFS="|"
set -- $line
unset IFS
mailto="$6" mailto="$6"
if [ "$6" != "&nbsp;" ] ; then if [ "$6" != "" ] ; then
mailto="<a href=\"mailto:$6\">$6</a>" mailto="<a href=\"mailto:$6\">$6</a>"
fi fi
@ -192,22 +251,23 @@ for i in `cat .logs | sort -t \\| +5`; do
echo "<td>$mailto</td>" >> $of echo "<td>$mailto</td>" >> $of
echo "<td><a href=\"http://${master}/errorlogs/${arch}-errorlogs/${baselogdir}/$1\">$2</a></td>" >> $of echo "<td><a href=\"http://${master}/errorlogs/${arch}-errorlogs/${baselogdir}/$1\">$2</a></td>" >> $of
affby=$3 affby="$3"
test $affby = "0" -o $affby = "-1" && affby="&nbsp;" test "${affby}" = "0" -o "${affby}" = "-1" && affby="&nbsp;"
echo "<td align=\"right\">$affby</td><td align=\"right\">$4 Kb</td>" >> $of echo "<td align=\"right\">${affby}</td><td align=\"right\">$4 Kb</td>" >> $of
echo "<td><a href=\"http://www.FreeBSD.org/cgi/cvsweb.cgi/ports/$5\">$5</a></td>" >> $of echo "<td><a href=\"http://www.FreeBSD.org/cgi/cvsweb.cgi/ports/$5\">$5</a></td>" >> $of
echo "<td>" >> $of echo "<td>" >> $of
test "$9" = "broken" && echo "<font color=\"red\">[B]</font>" >> $of test "$9" = "broken" && echo "<font color=\"red\">[B]</font>" >> $of
reason=$(echo $7 | tr '_' ' ') reason="$7"
echo "<a href=\"http://pointyhat.freebsd.org/#$8\">$reason</a>" >> $of echo "<a href=\"http://pointyhat.freebsd.org/#$8\">$reason</a>" >> $of
echo "</td>" >> $of echo "</td>" >> $of
date=$(echo ${10} | tr '_' ' ') date="${10}"
echo "<td>$date</td>" >> $of echo "<td>$date</td>" >> $of
echo "</tr>" >> $of echo "</tr>" >> $of
done done
fi
footer "" footer ""
mv -f $of index-maintainer.html mv -f $of index-maintainer.html
@ -217,10 +277,13 @@ mv -f $of index-maintainer.html
# #
header "<th>Reason</th><th>Port</th><th>Aff.</th><th>Size</th><th>CVS</th><th>Maintainer</th><th>Build date</th>" header "<th>Reason</th><th>Port</th><th>Aff.</th><th>Size</th><th>CVS</th><th>Maintainer</th><th>Build date</th>"
for i in `cat .logs | sort -t \\| +7`; do if [ ${#alllogs} -gt 0 ]; then
set $(echo "$i" | tr \| " " | sed -e "s@NONE@\\&nbsp;@g") sort -t \| +7 .logs/.all | while read line; do
IFS="|"
set -- ${line}
unset IFS
mailto="$6" mailto="$6"
if [ "$6" != "&nbsp;" ] ; then if [ "$6" != "" ] ; then
mailto="<a href=\"mailto:$6\">$6</a>" mailto="<a href=\"mailto:$6\">$6</a>"
fi fi
@ -228,23 +291,24 @@ for i in `cat .logs | sort -t \\| +7`; do
echo "<td>" >> $of echo "<td>" >> $of
test "$9" = "broken" && echo "<font color=\"red\">[B]</font>" >> $of test "$9" = "broken" && echo "<font color=\"red\">[B]</font>" >> $of
reason=$(echo $7 | tr '_' ' ') reason="$7"
echo "<a href=\"http://pointyhat.freebsd.org/#$8\">$reason</a>" >> $of echo "<a href=\"http://pointyhat.freebsd.org/#$8\">$reason</a>" >> $of
echo "</td>" >> $of echo "</td>" >> $of
echo "<td><a href=\"http://${master}/errorlogs/${arch}-errorlogs/${baselogdir}/$1\">$2</a></td>" >> $of echo "<td><a href=\"http://${master}/errorlogs/${arch}-errorlogs/${baselogdir}/$1\">$2</a></td>" >> $of
affby=$3 affby="$3"
test $affby = "0" -o $affby = "-1" && affby="&nbsp;" test "${affby}" = "0" -o "${affby}" = "-1" && affby="&nbsp;"
echo "<td align=\"right\">$affby</td><td align=\"right\">$4 Kb</td>" >> $of echo "<td align=\"right\">${affby}</td><td align=\"right\">$4 Kb</td>" >> $of
echo "<td><a href=\"http://www.FreeBSD.org/cgi/cvsweb.cgi/ports/$5\">$5</a></td>" >> $of echo "<td><a href=\"http://www.FreeBSD.org/cgi/cvsweb.cgi/ports/$5\">$5</a></td>" >> $of
echo "<td>$mailto</td>" >> $of echo "<td>$mailto</td>" >> $of
date=$(echo ${10} | tr '_' ' ') date="${10}"
echo "<td>$date</td>" >> $of echo "<td>$date</td>" >> $of
echo "</tr>" >> $of echo "</tr>" >> $of
done done
fi
footer "" footer ""
mv -f $of index-reason.html mv -f $of index-reason.html
@ -254,50 +318,49 @@ mv -f $of index-reason.html
# #
header "<th>Build date</th><th>Port</th><th>Aff.</th><th>Size</th><th>CVS</th><th>Maintainer</th><th>Reason</th>" header "<th>Build date</th><th>Port</th><th>Aff.</th><th>Size</th><th>CVS</th><th>Maintainer</th><th>Reason</th>"
for i in `cat .logs | sort -t \\| +9`; do if [ ${#alllogs} -gt 0 ]; then
set $(echo "$i" | tr \| " " | sed -e "s@NONE@\\&nbsp;@g") sort -t \| +9 .logs/.all | while read line; do
IFS="|"
set -- ${line}
unset IFS
mailto="$6" mailto="$6"
if [ "$6" != "&nbsp;" ] ; then if [ "$6" != "" ] ; then
mailto="<a href=\"mailto:$6\">$6</a>" mailto="<a href=\"mailto:$6\">$6</a>"
fi fi
echo "<tr>" >> $of echo "<tr>" >> $of
date=$(echo ${10} | tr '_' ' ') date="${10}"
echo "<td>$date</td>" >> $of echo "<td>$date</td>" >> $of
echo "<td><a href=\"http://${master}/errorlogs/${arch}-errorlogs/${baselogdir}/$1\">$2</a></td>" >> $of echo "<td><a href=\"http://${master}/errorlogs/${arch}-errorlogs/${baselogdir}/$1\">$2</a></td>" >> $of
affby=$3 affby="$3"
test $affby = "0" -o $affby = "-1" && affby="&nbsp;" test "${affby}" = "0" -o "${affby}" = "-1" && affby="&nbsp;"
echo "<td align=\"right\">$affby</td><td align=\"right\">$4 Kb</td>" >> $of echo "<td align=\"right\">${affby}</td><td align=\"right\">$4 Kb</td>" >> $of
echo "<td><a href=\"http://www.FreeBSD.org/cgi/cvsweb.cgi/ports/$5\">$5</a></td>" >> $of echo "<td><a href=\"http://www.FreeBSD.org/cgi/cvsweb.cgi/ports/$5\">$5</a></td>" >> $of
echo "<td>$mailto</td>" >> $of echo "<td>$mailto</td>" >> $of
echo "<td>" >> $of echo "<td>" >> $of
test "$9" = "broken" && echo "<font color=\"red\">[B]</font>" >> $of test "$9" = "broken" && echo "<font color=\"red\">[B]</font>" >> $of
reason=$(echo $7 | tr '_' ' ') reason="$7"
echo "<a href=\"http://pointyhat.freebsd.org/#$8\">$reason</a>" >> $of echo "<a href=\"http://pointyhat.freebsd.org/#$8\">$reason</a>" >> $of
echo "</td>" >> $of echo "</td>" >> $of
echo "</tr>" >> $of echo "</tr>" >> $of
done done
fi
footer "" footer ""
mv -f $of index-builddate.html mv -f $of index-builddate.html
# echo 'processlogs: at '`date`', create maintainer list' # echo 'processlogs: at '`date`', create maintainer list'
# #
# Get list of maintainers. # Get list of maintainers.
# if [ ${#alllogs} -gt 0 ]; then
for i in `cat .logs | sort -t \\| +9`; do cut -f 6 -d \| .logs/.all | sort -fu > maintainers
set $(echo "$i" | tr \| " " | sed -e "s@NONE@\\&nbsp;@g") else
mailto="$6" cat /dev/null > maintainers
if [ "$6" != "&nbsp;" ] ; then
mailto="<a href=\"mailto:$6\">$6</a>"
fi fi
maints="$maints $6" # echo 'processlogs: at '`date`', done'
done
echo $maints | sed -e 's/ /\
/g' | sort -fu > maintainers