Zap some old/non-useful code

Add some new command-line options:
  * -continue : restart the build without rebuilding ports that have
                already failed
  * -nofinish : don't run the post-processing steps when the build finishes
  * -finish : run the post-processing steps only
Reorder some code sections appropriately

Tools/portbuild/scripts/dopackages

@@ -20,7 +20,7 @@ errorexit () {
 }
 
 usage () {
-  echo "usage: [-restart] [-ftp] [-cdrom] [-nodummy] [-nobuild] [-noindex] [-noduds] [-nocvsup] [-nocvs] [-noportscvs] [-norestr] [-plistcheck] branch date"
+  echo "usage: [-continue] [-nofinish] [-finish] [-restart] [-ftp] [-cdrom] [-nodummy] [-nobuild] [-noindex] [-noduds] [-nocvsup] [-nocvs] [-noportscvs] [-norestr] [-plistcheck] branch date"
   errorexit 1
 }
 
@@ -86,6 +86,7 @@ makeduds () {
   echo "duds diff:"
   diff ${pb}/${arch}/${branch}/duds.old ${pb}/${arch}/${branch}/duds
   cp -p ${pb}/${arch}/${branch}/duds ${pb}/${arch}/${branch}/duds.old
+  cp -p ${pb}/${arch}/${branch}/duds ${pb}/${arch}/${branch}/duds.orig
 }
 
 # usage: setupnode pb arch scripts branch me node md5 tmpdir
@@ -151,10 +152,10 @@ archiveports () {
   arch=$2
   branch=$3
 
-  echo "started archive of /${branch}/ports at $(date)"
+  echo "started archive of ${pb}/${arch}/${branch}/ports at $(date)"
   cd ${pb}/${arch}/${branch}
   tar --exclude CVS -czf ${pb}/${arch}/${branch}/tarballs/ports.tar.gz ports
-  echo "ended archive of /${branch}/ports at $(date)"
+  echo "ended archive of ${pb}/${arch}/${branch}/ports at $(date)"
 }
 
 # usage: generatemd5 pb branch
@@ -200,6 +201,9 @@ nodummy=0
 cdrom=0
 ftp=0
 restart=0
+cont=0
+finish=0
+nofinish=0
 
 # optional arguments
 while [ $# -gt 2 ]; do
@@ -237,9 +241,19 @@ while [ $# -gt 2 ]; do
     x-ftp)
       ftp=1
       ;;
+    x-continue)
+      cont=1
+      ;;
     x-restart)
       restart=1
       ;;
+    x-nofinish)
+      nofinish=1
+      ;;
+    x-finish)
+      nobuild=1
+      finish=1
+      ;;
     *)
       usage
       ;;
@@ -247,6 +261,12 @@ while [ $# -gt 2 ]; do
   shift
 done
 
+if [ "$restart" = 1 -o "$cont" = 1 -o "$finish" = 1 ]; then
+  skipstart=1
+else
+  skipstart=0
+fi
+
 # mandatory arguments
 branch=$1
 date=$2
@@ -276,7 +296,7 @@ fi
 export DISTDIR=${pb}/${arch}/${branch}/distfiles
 export PACKAGES=${pb}/${arch}/${branch}/packages
 
-if [ "$restart" = 0 ]; then
+if [ "$skipstart" = 0 ]; then
   if [ "$noportscvs" = 0 ]; then
     echo "================================================"
     echo "running cvs update -PAd on ${PORTSDIR}"
@@ -319,6 +339,7 @@ if [ "$restart" = 0 ]; then
   fi
 fi
 
+if [ "$nobuild" = 0 ]; then
   md5=$(/sbin/md5 ${pb}/${arch}/${branch}/tarballs/bindist.tar | awk '{print $4}')
   echo "================================================"
   echo "setting up nodes"
@@ -327,8 +348,9 @@ for node in $(awk '{print $1}' ${pb}/${arch}/mlist); do
     setupnode ${pb} ${arch} ${scripts} ${branch} ${me} ${node} ${md5} ${scratchdir} &
     sleep 2
   done
+fi
 
-if [ "$restart" = 0 ]; then
+if [ "$skipstart" = 0 ]; then
   restrictedlist ${pb} ${arch} ${scripts} ${branch} &
   sleep 2
   
@@ -339,7 +361,7 @@ fi
   
 wait
 
-if [ "$restart" = 0 ]; then
+if [ "$skipstart" = 0 ]; then
   checkindex ${pb} ${arch} ${branch}
   
   ${pb}/scripts/makeparallel ${arch} ${branch}
@@ -356,6 +378,7 @@ if [ "$restart" = 0 ]; then
   fi
   
   # hack to extend length of dependency chain to build quickports first
+  # XXX Does this actually work? - kk 
   for i in ${quickports}; do
     if [ -d $i ]; then
       quickpkg="$(cd $i; make package-name)${PKGSUFFIX}"
@@ -370,8 +393,8 @@ if [ "$restart" = 0 ]; then
   
   cd ${pb}/${arch}
   if [ "$nobuild" = 0 ]; then
-    mkdir -p ${branch}/distfiles
+    #mkdir -p ${branch}/distfiles
-    chown -R ${user} ${branch}/distfiles
+    #chown -R ${user} ${branch}/distfiles
 
     cd ${pb}/${arch}/${branch}/bak
     rm -rf errors logs packages old-errors
@@ -387,6 +410,7 @@ if [ "$restart" = 0 ]; then
 
     cd ${pb}/${arch}/${branch}
     mkdir -p packages/All
+
     mkdir -p ${pb}/${arch}/archive/errorlogs
     rm -rf ${pb}/${arch}/archive/errorlogs/e.${branch}.${date} ${pb}/${arch}/archive/errorlogs/e.${branch}.${shortdate}
     mkdir -p ${pb}/${arch}/archive/errorlogs/e.${branch}.${date}
@@ -397,7 +421,7 @@ if [ "$restart" = 0 ]; then
     mkdir -p ${pb}/${arch}/archive/errorlogs/a.${branch}.${date}
     ln -sf ${pb}/${arch}/archive/errorlogs/a.${branch}.${date} ${pb}/${arch}/${branch}/logs
     ln -sf a.${branch}.${date} ${pb}/${arch}/archive/errorlogs/a.${branch}.${shortdate}
-    chown -RL ${user} errors logs packages
+#    chown -RL ${user} errors logs packages
 
     echo "error logs in ${pb}/${arch}/archive/errorlogs/e.${branch}.${date}"
     cp -p ${pb}/${arch}/cvsdone ${pb}/${arch}/archive/errorlogs/e.${branch}.${date}/cvsdone
@@ -419,7 +443,15 @@ if [ "$restart" = 0 ]; then
 fi
 
 if [ "$nobuild" = 0 ]; then
-  find /tmp -name 'make*' -mmin +60 | xargs rm -f
+  cd ${pb}/${arch}/${branch}
+
+  if [ "$cont" = 1 ]; then 
+    find errors/ -name \*.log | sed -e 's,\.log$,,' -e 's,^errors/,,' >> duds.errors
+    cat duds duds.errors | sort -u > duds.new
+    mv duds.new duds
+  else
+    cp duds.orig duds
+  fi
 
   count=$(awk '{sum+=$2}END{print sum+NR/2}' ${pb}/${arch}/mlist | sed -e 's/\..*$//')
   cd ${pb}/${arch}/${branch}/packages/All
@@ -474,40 +506,11 @@ if [ "$nobuild" = 0 ]; then
   phase2end=$(date +%s)
   echo "phase 2 took $(date -u -j -r $(($phase2end - $phase2start)) | awk '{print $4}')"
 
-  rm Makefile
-#  if [ "$nodummy" = 0 ]; then
-#    rm ${dummypkgs}
-#  fi
-
-  ${pb}/scripts/chopindex ${pb}/${arch}/${branch}/ports/${INDEXFILE} ${pb}/${arch}/${branch}/packages > \
-    ${pb}/${arch}/${branch}/packages/INDEX
   echo $(ls -1 ${pb}/${arch}/${branch}/packages/All | wc -l) "packages built"
   echo $(cat ${pb}/${arch}/${branch}/packages/${INDEXFILE} | wc -l) "lines in INDEX"
   echo $(echo $(du -sk ${pb}/${arch}/${branch}/packages | awk '{print $1}') / 1024 | bc) "MB of packages"
   echo $(echo $(du -sk ${pb}/${arch}/${branch}/distfiles | awk '{print $1}') / 1024 | bc) "MB of distfiles"
-
-  # Always delete restricted packages/distfiles since they're published on the
-  # website
-  echo "deleting restricted ports"
-  sh ${pb}/${arch}/${branch}/restricted.sh
-  ${pb}/scripts/chopindex ${pb}/${arch}/${branch}/ports/${INDEXFILE} ${pb}/${arch}/${branch}/packages > \
-    ${pb}/${arch}/${branch}/packages/INDEX
-  echo $(ls -1 ${pb}/${arch}/${branch}/packages/All | wc -l) "packages built"
-  echo $(cat ${pb}/${arch}/${branch}/packages/${INDEXFILE} | wc -l) "lines in INDEX"
-  echo $(echo $(du -sk ${pb}/${arch}/${branch}/packages | awk '{print $1}') / 1024 | bc) "MB of packages"
-  echo $(echo $(du -sk ${pb}/${arch}/${branch}/distfiles | awk '{print $1}') / 1024 | bc) "MB of distfiles"
-
-  rm -rf ${pb}/${arch}/${branch}/bad
-  mkdir -p ${pb}/${arch}/${branch}/bad
-  echo "checking packages"
-  for i in *${PKGSUFFIX}.; do
-    if ! ${PKGZIPCMD} -t $i; then
-      echo "Warning: package $i is bad, moving to ${pb}/${arch}/${branch}/bad"
-      # the latest link will be left behind...
-      mv $i ${pb}/${arch}/${branch}/bad
-      rm ../*/$i
 fi
-  done
 
 cd ${pb}/${arch}/${branch}
 if grep -qE '(ptimeout|pnohang): killing' make.1; then
@@ -515,81 +518,76 @@ if [ "$nobuild" = 0 ]; then
   grep -E '(ptimeout|pnohang): killing' make.1 | sed -e 's/^.*ptimeout:/ptimeout:/' -e 's/^.*pnohang:/pnohang:/'
 fi
 
+# Clean up temporary duds file
+if [ "$cont" = 1 ]; then                                               
+  cp duds.orig duds
+fi
+
+cd ${pb}/${arch}/${branch}/packages/All
+if [ "$nofinish" = 0 ]; then
+  rm -f Makefile
+
+  # Always delete restricted packages/distfiles since they're published on the
+  # website
+  echo "deleting restricted ports"
+  sh ${pb}/${arch}/${branch}/restricted.sh
+
+  if [ "$cdrom" = 1 ]; then
+    echo "deleting cdrom restricted ports"
+    sh ${pb}/${arch}/${branch}/cdrom.sh
+  fi
+
+  # Remove packages not listed in INDEX
+  ${pb}/scripts/prunepkgs ${pb}/${arch}/${branch}/ports/${INDEXFILE} ${pb}/${arch}/${branch}/packages
+fi
+
+#rm -rf ${pb}/${arch}/${branch}/bad
+#mkdir -p ${pb}/${arch}/${branch}/bad
+#echo "checking packages"
+#for i in *${PKGSUFFIX}; do
+#  if ! ${PKGZIPCMD} -t $i; then
+#    echo "Warning: package $i is bad, moving to ${pb}/${arch}/${branch}/bad"
+#    # the latest link will be left behind...
+#    mv $i ${pb}/${arch}/${branch}/bad
+#    rm ../*/$i
+#  fi
+#done
+
+if [ "$nofinish" = 0 ]; then
+  # Remove INDEX entries for packages that do not exist
+  ${pb}/scripts/chopindex ${pb}/${arch}/${branch}/ports/${INDEXFILE} ${pb}/${arch}/${branch}/packages > ${pb}/${arch}/${branch}/packages/INDEX
+
   cd ${pb}/${arch}/${branch}/old-errors
-  new=""
   for i in *.log; do
-    if [ ! -f ../errors/$i ]; then
+    cp -p ${i} ${pb}/${arch}/${branch}/errors/old-errors;
-      new="$new $(basename $i .log)"
-      cp -p ${i} ${pb}/${arch}/${branch}/errors/old-errors
-    fi
   done
-  if [ "x$new" != "x" ]; then
+
-    echo "The following port(s) didn't build the first time around: $new"
-  fi
   archiveports ${pb} ${arch} ${branch} &
   generatemd5 ${pb} ${arch} ${branch} &
-  sleep 5
-
-  echo "================================================"
-  echo "new failures"
-  echo "================================================"
-  cd ${pb}/${arch}/${branch}/errors
-  for i in *.log; do
-    if [ ! -f ${pb}/${arch}/${branch}/bak/errors/$i ]; then
-      echo -n " $(basename $i .log)"
-    fi
-  done
-
-  if cd ${pb}/${arch}/${branch}/bak/packages/All; then
-    echo
-    echo "================================================"
-    echo "old packages"
-    echo "================================================"
-    for i in *${PKGSUFFIX}; do
-      if [ ! -f ${pb}/${arch}/${branch}/packages/All/$i ]; then
-        echo -n " $(basename $i ${PKGSUFFIX})"
-      fi
-    done
-  fi
-
-  if cd ${pb}/${arch}/${branch}/bak/errors; then
-    echo
-    echo "================================================"
-    echo "old failures"
-    echo "================================================"
-    for i in *.log; do
-      if [ ! -f ${pb}/${arch}/${branch}/errors/$i ]; then
-        echo -n " $(basename $i .log)"
-      fi
-    done
-  fi
-
-  echo
-  echo "================================================"
-  echo "new packages"
-  echo "================================================"
-  cd ${pb}/${arch}/${branch}/packages/All
-  for i in *${PKGSUFFIX}; do
-    if [ ! -f ${pb}/${arch}/${branch}/bak/packages/All/$i ]; then
-      echo -n " $(basename $i ${PKGSUFFIX})"
-    fi
-  done
-  echo
   wait
 
   ls -asFlrt ${pb}/${arch}/${branch}/packages/All > ${pb}/${arch}/${branch}/logs/ls-lrt
   cp -p ${pb}/${arch}/${branch}/make.[01] ${pb}/${arch}/${branch}/logs
 
+  # Always delete restricted distfiles
+  echo "deleting restricted distfiles"
+  sh ${pb}/${arch}/${branch}/restricted.sh
+
+  if [ "$cdrom" = 1 ]; then
+    echo "deleting cdrom restricted distfiles"
+    sh ${pb}/${arch}/${branch}/cdrom.sh
+  fi
+
   echo "================================================"
   echo "copying distfiles"
   echo "================================================"
   echo "started at $(date)"
   cd ${pb}/${arch}
-  rm -rf ${branch}/bak/distfiles
+  #rm -rf ${branch}/bak/distfiles
-  mv -f ${branch}/distfiles ${branch}/bak
+  #mv -f ${branch}/distfiles ${branch}/bak
-  ${scripts}/dodistfiles ${pb}/${arch}/${branch}/bak/distfiles
+  ${scripts}/dodistfiles ${pb}/${arch}/${branch}/distfiles
-  rm -rf ${pb}/${arch}/${branch}/bak/distfiles/.btmp
+  rm -rf ${pb}/${arch}/${branch}/distfiles/.pbtmp
-  rm -f ${pb}/${arch}/${branch}/bak/distfiles/.done
+  rm -f ${pb}/${arch}/${branch}/distfiles/.done
 
   if [ "$branch" != "4-exp" ]; then
     # Currently broken - kk
@@ -599,7 +597,7 @@ if [ "$nobuild" = 0 ]; then
       echo "================================================"
       echo "copying packages"
       echo "================================================"
-      ${scripts}/docppackages ${branch}
+      ${scripts}/docppackages ${arch} ${branch}
     fi
   fi
 fi