From 2207d8843c3497b2d1c4f72d4bce90f0f8d11cef Mon Sep 17 00:00:00 2001 From: Tony Craig Date: Wed, 20 Oct 2021 08:23:37 -0700 Subject: [PATCH] Update test features: add create_fails script, QC process test, and wait on baseline results (#644) * add create_fails.csh and update documentation * update documentation * update qcnonbfb settings * add qc test capability and update bfbcomp logic to add check that the baseline results has completed * add PEND for bfbcomp result * update regression testing to use QC (instead of log compare) for qcchk test cases * update create_fails.csh * update create_fails log output --- cice.setup | 7 ++- .../scripts/machines/env.cheyenne_gnu | 7 +++ .../scripts/machines/env.cheyenne_intel | 7 +++ .../scripts/machines/env.cheyenne_pgi | 7 +++ .../scripts/options/set_nml.qc_nonbfb | 8 ---- .../scripts/options/set_nml.qcnonbfb | 16 +++++++ configuration/scripts/tests/QC/cice.t-test.py | 10 +++- .../scripts/tests/QC/gen_qc_cases.csh | 4 +- configuration/scripts/tests/baseline.script | 48 +++++++++++++++++++ configuration/scripts/tests/create_fails.csh | 24 ++++++++++ configuration/scripts/tests/decomp_suite.ts | 2 - configuration/scripts/tests/first_suite.ts | 1 + configuration/scripts/tests/prod_suite.ts | 8 ++-- configuration/scripts/tests/reprosum_suite.ts | 1 - configuration/scripts/tests/test_qcchk.script | 36 ++++++++++++++ .../scripts/tests/test_qcchkf.script | 36 ++++++++++++++ doc/source/developer_guide/dg_scripts.rst | 11 +++-- doc/source/user_guide/ug_testing.rst | 11 ++++- 18 files changed, 218 insertions(+), 26 deletions(-) delete mode 100644 configuration/scripts/options/set_nml.qc_nonbfb create mode 100644 configuration/scripts/options/set_nml.qcnonbfb create mode 100755 configuration/scripts/tests/create_fails.csh create mode 100644 configuration/scripts/tests/test_qcchk.script create mode 100644 configuration/scripts/tests/test_qcchkf.script diff --git a/cice.setup b/cice.setup index 3cae17d74..be9266dd2 100755 --- a/cice.setup +++ b/cice.setup @@ -480,6 +480,7 @@ else exit -1 endif cp -f ${ICE_SCRIPTS}/tests/report_results.csh ${tsdir} + cp -f ${ICE_SCRIPTS}/tests/create_fails.csh ${tsdir} cp -f ${ICE_SCRIPTS}/tests/poll_queue.csh ${tsdir} cat >! ${tsdir}/suite.submit << EOF0 @@ -919,7 +920,7 @@ EOF echo "ICE_GRID = ${grid} (${ICE_DECOMP_NXGLOB}x${ICE_DECOMP_NYGLOB}) blocksize=${ICE_DECOMP_BLCKX}x${ICE_DECOMP_BLCKY}x${ICE_DECOMP_MXBLCKS}" echo "ICE_DECOMP = ${ICE_DECOMP_DECOMP} ${ICE_DECOMP_DSHAPE}" if ($fbfbcomp != ${spval}) then - echo "ICE_BFBCOMP = ${fbfbcomp}" + echo "ICE_BFBCOMP = ${fbfbcomp}" endif #------------------------------------------------------------ @@ -1144,7 +1145,9 @@ if (\${dobuild} == true) then endif endif if (\${dosubmit} == true) then - ./cice.submit | tee -a ../suite.jobs + set jobid = \`./cice.submit\` + echo "\$jobid" + echo "\$jobid \${ICE_TESTNAME} " >> ../suite.jobs else if (\${dorun} == true) then ./cice.test endif diff --git a/configuration/scripts/machines/env.cheyenne_gnu b/configuration/scripts/machines/env.cheyenne_gnu index 3bfe59c31..8ddc443b1 100755 --- a/configuration/scripts/machines/env.cheyenne_gnu +++ b/configuration/scripts/machines/env.cheyenne_gnu @@ -29,6 +29,13 @@ if ($ICE_IOTYPE =~ pio*) then endif endif +if ($?ICE_TEST) then +if ($ICE_TEST =~ qcchk*) then + module load python + source /glade/u/apps/opt/ncar_pylib/ncar_pylib.csh default +endif +endif + endif limit coredumpsize unlimited diff --git a/configuration/scripts/machines/env.cheyenne_intel b/configuration/scripts/machines/env.cheyenne_intel index 4a430622e..28df6647d 100755 --- a/configuration/scripts/machines/env.cheyenne_intel +++ b/configuration/scripts/machines/env.cheyenne_intel @@ -29,6 +29,13 @@ if ($ICE_IOTYPE =~ pio*) then endif endif +if ($?ICE_TEST) then +if ($ICE_TEST =~ qcchk*) then + module load python + source /glade/u/apps/opt/ncar_pylib/ncar_pylib.csh default +endif +endif + endif limit coredumpsize unlimited diff --git a/configuration/scripts/machines/env.cheyenne_pgi b/configuration/scripts/machines/env.cheyenne_pgi index 693692842..d492129fb 100755 --- a/configuration/scripts/machines/env.cheyenne_pgi +++ b/configuration/scripts/machines/env.cheyenne_pgi @@ -29,6 +29,13 @@ if ($ICE_IOTYPE =~ pio*) then endif endif +if ($?ICE_TEST) then +if ($ICE_TEST =~ qcchk*) then + module load python + source /glade/u/apps/opt/ncar_pylib/ncar_pylib.csh default +endif +endif + endif limit coredumpsize unlimited diff --git a/configuration/scripts/options/set_nml.qc_nonbfb b/configuration/scripts/options/set_nml.qc_nonbfb deleted file mode 100644 index e34f75c35..000000000 --- a/configuration/scripts/options/set_nml.qc_nonbfb +++ /dev/null @@ -1,8 +0,0 @@ -npt = 87600 -dt = 1800.0 -dumpfreq = 'm' -dumpfreq_n = 12 -diagfreq = 24 -histfreq = 'd','x','x','x','x' -f_hi = 'd' -hist_avg = .false. diff --git a/configuration/scripts/options/set_nml.qcnonbfb b/configuration/scripts/options/set_nml.qcnonbfb new file mode 100644 index 000000000..a965b863c --- /dev/null +++ b/configuration/scripts/options/set_nml.qcnonbfb @@ -0,0 +1,16 @@ +dt = 3456.0 +npt_unit = 'y' +npt = 5 +year_init = 2005 +month_init = 1 +day_init = 1 +sec_init = 0 +use_leap_years = .false. +fyear_init = 2005 +ycycle = 1 +dumpfreq = 'm' +dumpfreq_n = 12 +diagfreq = 24 +histfreq = 'd','x','x','x','x' +f_hi = 'd' +hist_avg = .false. diff --git a/configuration/scripts/tests/QC/cice.t-test.py b/configuration/scripts/tests/QC/cice.t-test.py index 6f2c7e89b..c84583baa 100755 --- a/configuration/scripts/tests/QC/cice.t-test.py +++ b/configuration/scripts/tests/QC/cice.t-test.py @@ -177,7 +177,10 @@ def stage_one(data_d, num_files, mean_d, variance_d): df = n_eff - 1 # Read in t_crit table - nfid = nc.Dataset("configuration/scripts/tests/QC/CICE_t_critical_p0.8.nc", 'r') + if os.path.exists('./CICE_t_critical_p0.8.nc'): + nfid = nc.Dataset("./CICE_t_critical_p0.8.nc", 'r') + else: + nfid = nc.Dataset("configuration/scripts/tests/QC/CICE_t_critical_p0.8.nc", 'r') df_table = nfid.variables['df'][:] t_crit_table = nfid.variables['tcrit'][:] nfid.close() @@ -238,7 +241,10 @@ def stage_one(data_d, num_files, mean_d, variance_d): t_val = mean_d / np.sqrt(variance_d / num_files) # Find t_crit from the nearest value on the Lookup Table Test - nfid = nc.Dataset("configuration/scripts/tests/QC/CICE_Lookup_Table_p0.8_n1825.nc", 'r') + if os.path.exists('./CICE_Lookup_Table_p0.8_n1825.nc'): + nfid = nc.Dataset("./CICE_Lookup_Table_p0.8_n1825.nc", 'r') + else: + nfid = nc.Dataset("configuration/scripts/tests/QC/CICE_Lookup_Table_p0.8_n1825.nc", 'r') r1_table = nfid.variables['r1'][:] t_crit_table = nfid.variables['tcrit'][:] nfid.close() diff --git a/configuration/scripts/tests/QC/gen_qc_cases.csh b/configuration/scripts/tests/QC/gen_qc_cases.csh index 7a395f1c2..7d2db73b9 100755 --- a/configuration/scripts/tests/QC/gen_qc_cases.csh +++ b/configuration/scripts/tests/QC/gen_qc_cases.csh @@ -183,9 +183,9 @@ endif # Generate the non-BFB but non-climate-changing case echo "Generating nonbfb case" if ($testid != $spval) then - set result = `./cice.setup $options -s qc_nonbfb,long --testid qc_test_$testid | grep 'Test case dir\|already exists'` + set result = `./cice.setup $options -s qcnonbfb,long --testid qc_test_$testid | grep 'Test case dir\|already exists'` else - set result = `./cice.setup $options -s qc_nonbfb,long --testid qc_test | grep 'Test case dir\|already exists'` + set result = `./cice.setup $options -s qcnonbfb,long --testid qc_test | grep 'Test case dir\|already exists'` endif set nonbfb_dir = `echo "$result" | awk '{print$NF}'` if ($nonbfb_dir == "exists") then diff --git a/configuration/scripts/tests/baseline.script b/configuration/scripts/tests/baseline.script index 6f13807e3..ac69d49a0 100644 --- a/configuration/scripts/tests/baseline.script +++ b/configuration/scripts/tests/baseline.script @@ -36,6 +36,12 @@ if (${ICE_BASECOM} != ${ICE_SPVAL}) then ${ICE_CASEDIR}/casescripts/comparelog.csh ${base_file} ${test_file} notcicefile set bfbstatus = $status + else if (${ICE_TEST} =~ qcchk*) then + set test_dir = ${ICE_RUNDIR} + set base_dir = ${ICE_BASELINE}/${ICE_BASECOM}/${ICE_TESTNAME} + ${ICE_SANDBOX}/configuration/scripts/tests/QC/cice.t-test.py ${base_dir} ${test_dir} + set bfbstatus = $status + else set test_dir = ${ICE_RUNDIR}/restart @@ -116,6 +122,35 @@ endif if (${ICE_BFBCOMP} != ${ICE_SPVAL}) then + echo "PEND ${ICE_TESTNAME} bfbcomp ${ICE_BFBCOMP}" >> ${ICE_CASEDIR}/test_output + if (${ICE_BFBCOMP} != ${ICE_TESTNAME}) then + # Check if the baseline job is complete + set job = `grep " ${ICE_BFBCOMP} " ../suite.jobs | sed 's|^[^0-9]*\([0-9]*\).*$|\1|g'` + echo "checking on Job $job" + set qstatjob = 1 + set cnt = 0 + if (${job} =~ [0-9]*) then + while ($qstatjob) + ${ICE_MACHINE_QSTAT} $job >&/dev/null + set qstatus = $status +# echo $job $qstatus + if ($qstatus != 0) then + echo "Job $job completed" + set qstatjob = 0 + else + @ cnt = $cnt + 1 + echo "Waiting for $job to complete $cnt" + sleep 60 # Sleep for 1 minute, so as not to overwhelm the queue manager + if ($cnt > 100) then + echo "No longer waiting for $job to complete" + set qstatjob = 0 # Abandon check after 100 sleep 60 checks + endif + endif +# echo $qstatjob + end + endif + endif + if (${ICE_TEST} == "logbfb") then set test_file = `ls -1t ${ICE_RUNDIR}/cice.runlog* | head -1` set base_file = `ls -1t ${ICE_RUNDIR}/../${ICE_BFBCOMP}.${ICE_TESTID}/cice.runlog* | head -1` @@ -127,6 +162,16 @@ if (${ICE_BFBCOMP} != ${ICE_SPVAL}) then ${ICE_CASEDIR}/casescripts/comparelog.csh ${base_file} ${test_file} set bfbstatus = $status + + else if (${ICE_TEST} =~ qcchk*) then + set test_dir = ${ICE_RUNDIR} + set base_dir = ${ICE_RUNDIR}/../${ICE_BFBCOMP}.${ICE_TESTID} + ${ICE_SANDBOX}/configuration/scripts/tests/QC/cice.t-test.py ${base_dir} ${test_dir} + set bfbstatus = $status + # expecting failure, so switch value + if (${ICE_TEST} =~ qcchkf*) then + @ bfbstatus = 1 - $bfbstatus + endif else set test_dir = ${ICE_RUNDIR}/restart set base_dir = ${ICE_RUNDIR}/../${ICE_BFBCOMP}.${ICE_TESTID}/restart @@ -140,6 +185,9 @@ if (${ICE_BFBCOMP} != ${ICE_SPVAL}) then set bfbstatus = $status endif + mv -f ${ICE_CASEDIR}/test_output ${ICE_CASEDIR}/test_output.prev + cat ${ICE_CASEDIR}/test_output.prev | grep -iv "${ICE_TESTNAME} bfbcomp " >! ${ICE_CASEDIR}/test_output + rm -f ${ICE_CASEDIR}/test_output.prev if (${bfbstatus} == 0) then echo "PASS ${ICE_TESTNAME} bfbcomp ${ICE_BFBCOMP}" >> ${ICE_CASEDIR}/test_output echo "bfb baseline and test dataset are identical" diff --git a/configuration/scripts/tests/create_fails.csh b/configuration/scripts/tests/create_fails.csh new file mode 100755 index 000000000..d22fe01ad --- /dev/null +++ b/configuration/scripts/tests/create_fails.csh @@ -0,0 +1,24 @@ +#!/bin/csh + +echo " " +set tmpfile = create_fails.tmp +set outfile = fails.ts + +./results.csh >& /dev/null +cat results.log | grep ' run\| test' | grep -v "#" | grep -v PASS | cut -f 2 -d " " | sort -u >! $tmpfile + +echo "# Test Grid PEs Sets" >! $outfile +foreach line ( "`cat $tmpfile`" ) + #echo $line + set test = `echo $line | cut -d "_" -f 3` + set grid = `echo $line | cut -d "_" -f 4` + set pes = `echo $line | cut -d "_" -f 5` + set opts = `echo $line | cut -d "_" -f 6- | sed 's/_/,/g'` + echo "$test $grid $pes $opts" >> $outfile +end + +rm $tmpfile +echo "$0 done" +echo "Failed tests can be rerun with the test suite file...... $outfile" +echo "To run a new test suite, copy $outfile to the top directory and do something like" +echo " ./cice.setup --suite $outfile ..." diff --git a/configuration/scripts/tests/decomp_suite.ts b/configuration/scripts/tests/decomp_suite.ts index 9c82c5d27..c39c3ddfe 100644 --- a/configuration/scripts/tests/decomp_suite.ts +++ b/configuration/scripts/tests/decomp_suite.ts @@ -3,7 +3,6 @@ restart gx3 4x2x25x29x4 dslenderX2 restart gx1 64x1x16x16x10 dwghtfile restart gbox180 16x1x6x6x60 dspacecurve,debugblocks decomp gx3 4x2x25x29x5 none -sleep 30 restart gx3 1x1x50x58x4 droundrobin,thread restart_gx3_4x2x25x29x4_dslenderX2 restart gx3 4x1x25x116x1 dslenderX1,thread restart_gx3_4x2x25x29x4_dslenderX2 restart gx3 6x2x4x29x18 dspacecurve restart_gx3_4x2x25x29x4_dslenderX2 @@ -27,7 +26,6 @@ restart gx3 8x1x25x29x4 drakeX2,thread restart_gx3_4x2x25x2 smoke gx3 4x2x25x29x4 debug,run2day,dslenderX2 smoke gx1 64x1x16x16x10 debug,run2day,dwghtfile smoke gbox180 16x1x6x6x60 debug,run2day,dspacecurve,debugblocks -sleep 30 smoke gx3 1x1x25x58x8 debug,run2day,droundrobin,thread smoke_gx3_4x2x25x29x4_debug_dslenderX2_run2day smoke gx3 20x1x5x116x1 debug,run2day,dslenderX1,thread smoke_gx3_4x2x25x29x4_debug_dslenderX2_run2day smoke gx3 6x2x4x29x18 debug,run2day,dspacecurve smoke_gx3_4x2x25x29x4_debug_dslenderX2_run2day diff --git a/configuration/scripts/tests/first_suite.ts b/configuration/scripts/tests/first_suite.ts index bf6c813f6..31eba9fb7 100644 --- a/configuration/scripts/tests/first_suite.ts +++ b/configuration/scripts/tests/first_suite.ts @@ -1,5 +1,6 @@ # Test Grid PEs Sets BFB-compare smoke gx3 8x2 diag1,run5day restart gx3 4x2x25x29x4 dslenderX2 +smoke gx3 4x2x25x29x4 debug,run2day,dslenderX2 logbfb gx3 4x2x25x29x4 dslenderX2,diag1,reprosum smoke gx3 1x2 run2day diff --git a/configuration/scripts/tests/prod_suite.ts b/configuration/scripts/tests/prod_suite.ts index 8793dfed2..04982adb1 100644 --- a/configuration/scripts/tests/prod_suite.ts +++ b/configuration/scripts/tests/prod_suite.ts @@ -1,4 +1,6 @@ # Test Grid PEs Sets BFB-compare -smoke gx1 64x1 qc,medium -smoke gx1 64x2 gx1prod,long,run10year - +qcchk gx3 72x1 qc,medium qcchk_gx3_72x1_medium_qc +qcchk gx1 144x1 qc,medium +smoke gx1 144x2 gx1prod,long,run10year +qcchkf gx3 72x1 qc,medium,alt02 qcchk_gx3_72x1_medium_qc +qcchk gx3 72x1 qcnonbfb,medium qcchk_gx3_72x1_medium_qc diff --git a/configuration/scripts/tests/reprosum_suite.ts b/configuration/scripts/tests/reprosum_suite.ts index dd6a6d56b..a7f3fe5bc 100644 --- a/configuration/scripts/tests/reprosum_suite.ts +++ b/configuration/scripts/tests/reprosum_suite.ts @@ -1,7 +1,6 @@ # Test Grid PEs Sets BFB-compare logbfb gx3 4x2x25x29x4 dslenderX2,diag1,reprosum #logbfb gx3 4x2x25x29x4 dslenderX2,diag1 -sleep 60 logbfb gx3 1x1x50x58x4 droundrobin,diag1,thread,maskhalo,reprosum logbfb_gx3_4x2x25x29x4_diag1_dslenderX2_reprosum logbfb gx3 4x1x25x116x1 dslenderX1,diag1,thread,maskhalo,reprosum logbfb_gx3_4x2x25x29x4_diag1_dslenderX2_reprosum logbfb gx3 1x20x5x29x80 dsectrobin,diag1,short,reprosum logbfb_gx3_4x2x25x29x4_diag1_dslenderX2_reprosum diff --git a/configuration/scripts/tests/test_qcchk.script b/configuration/scripts/tests/test_qcchk.script new file mode 100644 index 000000000..81b5f05fc --- /dev/null +++ b/configuration/scripts/tests/test_qcchk.script @@ -0,0 +1,36 @@ + +cp ${ICE_SANDBOX}/configuration/scripts/tests/QC/CICE_t_critical_p0.8.nc . +cp ${ICE_SANDBOX}/configuration/scripts/tests/QC/CICE_Lookup_Table_p0.8_n1825.nc . + +#---------------------------------------------------- +# Run the CICE model +# cice.run returns -1 if run did not complete successfully + +./cice.run +set res="$status" + +set log_file = `ls -t1 ${ICE_RUNDIR}/cice.runlog* | head -1` +set ttimeloop = `grep TimeLoop ${log_file} | grep Timer | cut -c 22-32` +set tdynamics = `grep Dynamics ${log_file} | grep Timer | cut -c 22-32` +set tcolumn = `grep Column ${log_file} | grep Timer | cut -c 22-32` +if (${ttimeloop} == "") set ttimeloop = -1 +if (${tdynamics} == "") set tdynamics = -1 +if (${tcolumn} == "") set tcolumn = -1 + +mv -f ${ICE_CASEDIR}/test_output ${ICE_CASEDIR}/test_output.prev +cat ${ICE_CASEDIR}/test_output.prev | grep -iv "${ICE_TESTNAME} run" >! ${ICE_CASEDIR}/test_output +mv -f ${ICE_CASEDIR}/test_output ${ICE_CASEDIR}/test_output.prev +cat ${ICE_CASEDIR}/test_output.prev | grep -iv "${ICE_TESTNAME} test" >! ${ICE_CASEDIR}/test_output +rm -f ${ICE_CASEDIR}/test_output.prev + +set grade = PASS +if ( $res != 0 ) then + set grade = FAIL + echo "$grade ${ICE_TESTNAME} run ${ttimeloop} ${tdynamics} ${tcolumn}" >> ${ICE_CASEDIR}/test_output + echo "$grade ${ICE_TESTNAME} test " >> ${ICE_CASEDIR}/test_output + exit 99 +endif + +echo "$grade ${ICE_TESTNAME} run ${ttimeloop} ${tdynamics} ${tcolumn}" >> ${ICE_CASEDIR}/test_output +echo "$grade ${ICE_TESTNAME} test " >> ${ICE_CASEDIR}/test_output + diff --git a/configuration/scripts/tests/test_qcchkf.script b/configuration/scripts/tests/test_qcchkf.script new file mode 100644 index 000000000..81b5f05fc --- /dev/null +++ b/configuration/scripts/tests/test_qcchkf.script @@ -0,0 +1,36 @@ + +cp ${ICE_SANDBOX}/configuration/scripts/tests/QC/CICE_t_critical_p0.8.nc . +cp ${ICE_SANDBOX}/configuration/scripts/tests/QC/CICE_Lookup_Table_p0.8_n1825.nc . + +#---------------------------------------------------- +# Run the CICE model +# cice.run returns -1 if run did not complete successfully + +./cice.run +set res="$status" + +set log_file = `ls -t1 ${ICE_RUNDIR}/cice.runlog* | head -1` +set ttimeloop = `grep TimeLoop ${log_file} | grep Timer | cut -c 22-32` +set tdynamics = `grep Dynamics ${log_file} | grep Timer | cut -c 22-32` +set tcolumn = `grep Column ${log_file} | grep Timer | cut -c 22-32` +if (${ttimeloop} == "") set ttimeloop = -1 +if (${tdynamics} == "") set tdynamics = -1 +if (${tcolumn} == "") set tcolumn = -1 + +mv -f ${ICE_CASEDIR}/test_output ${ICE_CASEDIR}/test_output.prev +cat ${ICE_CASEDIR}/test_output.prev | grep -iv "${ICE_TESTNAME} run" >! ${ICE_CASEDIR}/test_output +mv -f ${ICE_CASEDIR}/test_output ${ICE_CASEDIR}/test_output.prev +cat ${ICE_CASEDIR}/test_output.prev | grep -iv "${ICE_TESTNAME} test" >! ${ICE_CASEDIR}/test_output +rm -f ${ICE_CASEDIR}/test_output.prev + +set grade = PASS +if ( $res != 0 ) then + set grade = FAIL + echo "$grade ${ICE_TESTNAME} run ${ttimeloop} ${tdynamics} ${tcolumn}" >> ${ICE_CASEDIR}/test_output + echo "$grade ${ICE_TESTNAME} test " >> ${ICE_CASEDIR}/test_output + exit 99 +endif + +echo "$grade ${ICE_TESTNAME} run ${ttimeloop} ${tdynamics} ${tcolumn}" >> ${ICE_CASEDIR}/test_output +echo "$grade ${ICE_TESTNAME} test " >> ${ICE_CASEDIR}/test_output + diff --git a/doc/source/developer_guide/dg_scripts.rst b/doc/source/developer_guide/dg_scripts.rst index 50853b3ea..dac5e9a52 100644 --- a/doc/source/developer_guide/dg_scripts.rst +++ b/doc/source/developer_guide/dg_scripts.rst @@ -78,7 +78,8 @@ are the three scripts that modify **ice_in** and **cice.settings**. To add new options, just add new files to the **configurations/scripts/options/** directory with appropriate names and syntax. The set_nml file syntax is the same as namelist syntax and the set_env files are consistent with csh setenv syntax. See other files for -examples of the syntax. +examples of the syntax. The name of the option (i.e. diag1, debug, bgcISPOL) should not +have any special characters in the name as this can impact scripts usage. .. _build: @@ -163,7 +164,7 @@ it's working properly. .. _dev_validation: -Code Validation Script +QC Process Validation ---------------------- The code validation (aka QC or quality control) test validates non bit-for-bit model changes. The directory @@ -193,9 +194,9 @@ to the ``cice.setup`` script. These options include: * ``--queue`` : Queue for the batch submission * ``--testid`` : test ID, user-defined id for testing -The script creates 4 test cases, with testIDs ``qc_base``, ``qc_bfb``, ``qc_nonbfb``, +The script creates 4 test cases, with testIDs ``qc_base``, ``qc_bfb``, ``qc_test``, and ``qc_fail``. ``qc_base`` is the base test case with the default QC namelist. -``qc_bfb`` is identical to ``qc_base``. ``qc_nonbfb`` is a test that is not bit-for-bit +``qc_bfb`` is identical to ``qc_base``. ``qc_test`` is a test that is not bit-for-bit when compared to ``qc_base``, but not climate changing. ``qc_fail`` is a test that is not bit-for-bit and also climate changing. @@ -222,7 +223,7 @@ To perform the QC validation, execute the following commands. # From the CICE base directory cp configuration/scripts/tests/QC/gen_qc_cases.csh . - cp configuration/scripts/tests/QC/compare_qc_cases.csh + cp configuration/scripts/tests/QC/compare_qc_cases.csh . # Create the required test cases ./gen_qc_cases.csh -m --acct diff --git a/doc/source/user_guide/ug_testing.rst b/doc/source/user_guide/ug_testing.rst index 955328bf5..005f4f851 100644 --- a/doc/source/user_guide/ug_testing.rst +++ b/doc/source/user_guide/ug_testing.rst @@ -297,6 +297,13 @@ results.csh script in the testsuite.[testid]:: cd testsuite.[testid] ./results.csh +The script **create_fails.csh** will process the output from results.csh and generate a new +test suite file, **fails.ts**, from the failed tests. +**fails.ts** can then be edited and passed into ``cice.setup --suite fails.ts ...`` to rerun +subsets of failed tests to more efficiently move thru the development, testing, and +validation process. However, a full test suite should be run on the final development +version of the code. + To report the test results, as is required for Pull Requests to be accepted into the master the CICE Consortium code see :ref:`testreporting`. @@ -411,8 +418,10 @@ The *cice.setup** options ``--setup-only``, ``--setup-build``, and ``--setup-bui which means by default the test suite builds and submits the jobs. By defining other values for those environment variables, users can control the suite script. When using **suite.submit** manually, the string ``true`` (all lowercase) is the only string that will turn on a feature, and both SUITE_RUN and SUITE_SUBMIT cannot be true at the same time. -By leveraging the **cice.setup** command line arguments ``--setup-only``, ``--setup-build``, and ``--setup-build-run`` as well as the environment variables SUITE_BUILD, SUITE_RUN, and SUITE_SUBMIT, users can run **cice.setup** and **suite.submit** in various combinations to quickly setup, setup and build, submit, resubmit, run interactively, or rebuild and resubmit full testsuites quickly and easily. See below for an example. +By leveraging the **cice.setup** command line arguments ``--setup-only``, ``--setup-build``, and ``--setup-build-run`` as well as the environment variables SUITE_BUILD, SUITE_RUN, and SUITE_SUBMIT, users can run **cice.setup** and **suite.submit** in various combinations to quickly setup, setup and build, submit, resubmit, run interactively, or rebuild and resubmit full testsuites quickly and easily. See :ref:`examplesuites` for an example. + +.. _examplesuites: Test Suite Examples ~~~~~~~~~~~~~~~~~~~~~~~~