update test reporting (#310)

configuration/scripts/icepack.test.setup.csh

@@ -40,6 +40,13 @@ if ( ! -f ${ICE_RUNDIR}/icepack ) then
   exit 99
 endif
 
+# Initial test results and Reset test results for rerun
+mv -f ${ICE_CASEDIR}/test_output ${ICE_CASEDIR}/test_output.prev
+echo "#---" >! ${ICE_CASEDIR}/test_output
+cat ${ICE_CASEDIR}/test_output.prev | grep -i "${ICE_TESTNAME} build" >> ${ICE_CASEDIR}/test_output
+echo "PEND ${ICE_TESTNAME} run" >> ${ICE_CASEDIR}/test_output
+rm -f ${ICE_CASEDIR}/test_output.prev
+
 EOF2
 
 if ( -f ${ICE_SCRIPTS}/tests/test_${ICE_TEST}.script) then

configuration/scripts/tests/report_results.csh

@@ -1,5 +1,19 @@
 #!/bin/csh -f
 
+if ($#argv == 0) then
+  echo "${0}: Running results.csh"
+  ./results.csh >& /dev/null
+else if ($#argv == 1) then
+  if ("$argv[1]" =~ "-n") then
+    #continue
+  else
+    echo "$0 usage:"
+    echo "$0 [-n]"
+    echo "   -n : do NOT run results.csh (by default it does)"
+    exit -1
+  endif
+endif
+
 if (! -e results.log) then
   echo " "
   echo "${0}: ERROR results.log does not exist, try running results.csh"
@@ -79,13 +93,21 @@ unset noglob
 
 foreach compiler ( ${compilers} )
 
-  set ofile = "${shhash}.${mach}.${compiler}.${xcdat}.${xctim}"
-  set outfile = "${wikiname}/${tsubdir}/${ofile}.md"
+  set cnt = 0
+  set found = 1
+  while ($found == 1)
+    set ofile = "${shhash}.${mach}.${compiler}.${xcdat}.${xctim}.$cnt"
+    set outfile = "${wikiname}/${tsubdir}/${ofile}.md"
+    if (-e ${outfile}) then
+      @ cnt = $cnt + 1
+    else
+      set found = 0
+    endif
+  end
+
   mkdir -p ${wikiname}/${tsubdir}
   echo "${0}: writing to ${outfile}"
 
-  if (-e ${outfile}) rm -f ${outfile}
-
 cat >! ${outfile} << EOF
 
 |Bld|Run|Test| Regr | Compare | Timing | Case |
@@ -103,7 +125,7 @@ EOF
 foreach case ( ${cases} )
 if ( ${case} =~ *_${compiler}_* ) then
 
-# check thata case results are meaningful
+# check that case results are meaningful
   set fbuild = `grep " ${case} " results.log | grep " build"   | cut -c 1-4`
   set frun   = `grep " ${case} " results.log | grep " run"     | cut -c 1-4`
   set ftest  = `grep " ${case} " results.log | grep " test"    | cut -c 1-4`
@@ -115,8 +137,8 @@ if ( $fbuild != "" || $frun != "" || $ftest != "" ) then
   set ftest  = `grep " ${case} " results.log | grep " test"    | cut -c 1-4`
   set fregr  = `grep " ${case} " results.log | grep " compare" | cut -c 1-4`
   set fcomp  = `grep " ${case} bfbcomp " results.log | cut -c 1-4`
-  if (${ftest}  == "PASS") set frun   = "PASS"
-  if (${frun}   == "PASS") set fbuild = "PASS"
+#  if (${ftest}  == "PASS") set frun   = "PASS"
+#  if (${frun}   == "PASS") set fbuild = "PASS"
 
   set vregr  = `grep " ${case} " results.log | grep " compare" | cut -d " " -f 4 | sed 's/\./ /g' `
   set vcomp  = `grep " ${case} bfbcomp " results.log | cut -d " " -f 4`
@@ -145,7 +167,7 @@ if ( $fbuild != "" || $frun != "" || $ftest != "" ) then
   if (${fregr}  == "FAIL") set rregr  = ${red}
   if (${fcomp}  == "FAIL") set rcomp  = ${red}
 
-  if (${fbuild} == "") set rbuild = ${red}
+  if (${fbuild} == "") set rbuild = ${gray}
   if (${frun}   == "") set rrun   = ${red}
   if (${ftest}  == "") set rtest  = ${red}
   if (${fregr}  == "") set rregr  = ${gray}

configuration/scripts/tests/test_restart.script

@@ -7,14 +7,6 @@ cp icepack_in icepack_in.0
 ${ICE_CASEDIR}/casescripts/parse_namelist.sh icepack_in ${ICE_CASEDIR}/casescripts/test_nml.restart1
 cp icepack_in icepack_in.1
 
-mv -f ${ICE_CASEDIR}/test_output ${ICE_CASEDIR}/test_output.prev
-cat ${ICE_CASEDIR}/test_output.prev | grep -iv "${ICE_TESTNAME} run" >! ${ICE_CASEDIR}/test_output
-mv -f ${ICE_CASEDIR}/test_output ${ICE_CASEDIR}/test_output.prev
-cat ${ICE_CASEDIR}/test_output.prev | grep -iv "${ICE_TESTNAME} test" >! ${ICE_CASEDIR}/test_output
-rm -f ${ICE_CASEDIR}/test_output.prev
-echo "RUN  ${ICE_TESTNAME} run " >> ${ICE_CASEDIR}/test_output
-echo "PEND ${ICE_TESTNAME} test " >> ${ICE_CASEDIR}/test_output
-
 ./icepack.run
 set res="$?"
 

configuration/scripts/tests/test_smoke.script

@@ -3,11 +3,6 @@
 # Run the ICEPACK model
 # icepack.run returns -1 if run did not complete successfully
 
-mv -f ${ICE_CASEDIR}/test_output ${ICE_CASEDIR}/test_output.prev
-cat ${ICE_CASEDIR}/test_output.prev | grep -iv "${ICE_TESTNAME} run" >! ${ICE_CASEDIR}/test_output
-rm -f ${ICE_CASEDIR}/test_output.prev
-echo "RUN  ${ICE_TESTNAME} run " >> ${ICE_CASEDIR}/test_output
-
 ./icepack.run
 set res="$?"
 

doc/source/user_guide/ug_testing.rst

@@ -481,7 +481,11 @@ To post results, once a test suite is complete, run ``results.csh`` and
   ./results.csh
   ./report_results.csh
 
-The reporting can also be automated by adding ``--report``
+``report_results.csh`` will run ``results.csh`` by default automatically, but
+we recommmend running it manually first to verify results before publishing
+them.  ``report_results.csh -n`` will turn off automatic running of ``results.csh``.
+
+The reporting can also be automated in a test suite by adding ``--report`` to ``icepack.setup``
 ::
 
   ./icepack.setup --suite base_suite --mach conrad --env cray --testid v01a --report

icepack.setup

@@ -792,10 +792,6 @@ EOF2
         exit -1
       endif
 
-      # Initial test_output file
-      echo "#---" >! test_output
-      echo "PEND ${testname_noid} " >> test_output
-
     endif
 
     #------------------------------------------------------------
@@ -837,24 +833,55 @@ if ( ${dosuite} == 1 ) then
   # Add code to results.csh to count the number of failures
 cat >> ${tsdir}/results.csh << EOF
 cat ./results.log
-set pends = \`cat ./results.log | grep PEND | wc -l\`
-set failures = \`cat ./results.log | grep FAIL | wc -l\`
-set success = \`cat ./results.log | grep PASS | wc -l\`
-set comments = \`cat ./results.log | grep "#" | wc -l\`
-set alltotal = \`cat ./results.log | wc -l\`
+set pends     = \`cat ./results.log | grep PEND | wc -l\`
+set misses    = \`cat ./results.log | grep MISS | wc -l\`
+set failures  = \`cat ./results.log | grep FAIL | wc -l\`
+set failbuild = \`cat ./results.log | grep FAIL | grep " build " | wc -l\`
+set failrun   = \`cat ./results.log | grep FAIL | grep " run " | wc -l\`
+set failtest  = \`cat ./results.log | grep FAIL | grep " test " | wc -l\`
+set failcomp  = \`cat ./results.log | grep FAIL | grep " compare " | wc -l\`
+set failbfbc  = \`cat ./results.log | grep FAIL | grep " bfbcomp " | wc -l\`
+set failgen   = \`cat ./results.log | grep FAIL | grep " generate " | wc -l\`
+set success   = \`cat ./results.log | grep 'PASS\|COPY' | wc -l\`
+set comments  = \`cat ./results.log | grep "#" | wc -l\`
+set alltotal  = \`cat ./results.log | wc -l\`
 @ total = \$alltotal - \$comments
+@ chkcnt = \$pends + \$misses + \$failures + \$success
 
 echo "#------- " >> results.log
 echo " " >> results.log
-echo "#totl = \$total" >> results.log
+echo "#totl = \$total total" >> results.log
+echo "#chkd = \$chkcnt checked" >> results.log
 echo "#pass = \$success" >> results.log
-echo "#fail = \$failures" >> results.log
 echo "#pend = \$pends" >> results.log
+echo "#miss = \$misses" >> results.log
+echo "#fail = \$failures" >> results.log
+echo "    #failbuild = \$failbuild" >> results.log
+echo "    #failrun   = \$failrun" >> results.log
+echo "    #failtest  = \$failtest" >> results.log
+echo "    #failcomp  = \$failcomp" >> results.log
+echo "    #failbfbc  = \$failbfbc" >> results.log
+echo "    #failgen   = \$failgen" >> results.log
 
 echo ""
-echo "\$success of \$total tests PASSED"
-echo "\$failures of \$total tests FAILED"
-echo "\$pends of \$total tests PENDING"
+echo "Descriptors:"
+echo " PASS - successful completion"
+echo " COPY - previously compiled code was copied for new test"
+echo " MISS - comparison data is missing"
+echo " PEND - status is undertermined; test may still be queued, running, or timed out"
+echo " FAIL - test failed"
+echo ""
+echo "\$chkcnt measured results of \$total total results"
+echo "\$success of \$chkcnt tests PASSED"
+echo "\$pends of \$chkcnt tests PENDING"
+echo "\$misses of \$chkcnt tests MISSING data"
+echo "\$failures of \$chkcnt tests FAILED"
+#echo "  \$failbuild of \$failures FAILED build"
+#echo "  \$failrun of \$failures FAILED run"
+#echo "  \$failtest of \$failures FAILED test"
+#echo "  \$failcomp of \$failures FAILED compare"
+#echo "  \$failbfbc of \$failures FAILED bfbcomp"
+#echo "  \$failgen of \$failures FAILED generate"
 exit \$failures
 EOF