From 0698e1a047b8a007f650236345fa4fd33c7e4d32 Mon Sep 17 00:00:00 2001 From: Anthony Craig Date: Fri, 10 Apr 2020 17:47:55 -0600 Subject: [PATCH 01/27] add .codecov.yml --- .codecov.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .codecov.yml diff --git a/.codecov.yml b/.codecov.yml new file mode 100644 index 000000000..0058c5f0b --- /dev/null +++ b/.codecov.yml @@ -0,0 +1,2 @@ +# do not comment with code coverage metrics in github pull requests +comment: false From e6705c4b15de60085a60013d65bb7332567b44f9 Mon Sep 17 00:00:00 2001 From: Anthony Craig Date: Fri, 10 Apr 2020 18:03:06 -0600 Subject: [PATCH 02/27] update icepack.setup --- icepack.setup | 86 +++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 77 insertions(+), 9 deletions(-) diff --git a/icepack.setup b/icepack.setup index 2f54cfa2b..07c25ca56 100755 --- a/icepack.setup +++ b/icepack.setup @@ -34,6 +34,8 @@ set stime = `date -u "+%H%M%S"` set docase = 0 set dotest = 0 set dosuite = 0 +set codecov = 0 # code coverage measurement and reporting +set codecovflag = false if ($#argv < 1) then set helpheader = 1 @@ -80,7 +82,7 @@ SYNOPSIS --suite SUITE[,SUITE2] -m MACH --testid ID [-e ENV1,ENV2][--acct ACCT][--bdir DIR][--bgen DIR] - [--bcmp DIR][--tdir PATH][--report] + [--bcmp DIR][--tdir PATH][--report][--codecov] DESCRIPTION --help, -h : help @@ -107,6 +109,8 @@ DESCRIPTION --testid : test ID, user-defined id for testing (REQUIRED with --test or --suite) --diff : generate comparison against another case --report : automatically post results when tests are complete + --codecov : generate and report test coverage metrics when tests are complete, + requires GNU compiler (--env gnu) EXAMPLES icepack.setup -c caseB -m gordon -e cray -s diag1,debug @@ -235,15 +239,20 @@ while (1) set report = 1 shift argv + else if ("$option" == "--codecov") then + set codecov = 1 + set codecovflag = true + shift argv + # arguments with settings else shift argv if ( $#argv < 1 ) then - echo "${0}: ERROR1 in $option" + echo "${0}: ERROR in $option, unsupported or missing an argument" exit -1 endif - if ($argv[1] =~ $dash* ) then - echo "${0}: ERROR2 in $option" + if ("$argv[1]" =~ "$dash*" ) then + echo "${0}: ERROR in $option, possibly missing an argument" exit -1 endif @@ -308,7 +317,30 @@ if (${dosum} > 1) then exit -1 endif +if ($codecov == 1 && $report == 1) then + echo "${0}: ERROR in arguments, not recommmended to set both --codecov and --report" + exit -1 +endif + +if ($codecov == 1 && "$compilers" != "gnu") then + echo "${0}: ERROR in arguments, must use --env gnu with --codecov" + exit -1 +endif + +if ($codecov == 1 && `where curl` == "" && `where wget` == "") then + echo "${0}: ERROR 'curl' or 'wget' is required for --codecov" + exit -1 +endif + if (${dosuite} == 0) then + if ($report == 1) then + echo "${0}: ERROR in arguments, must use --suite with --report" + exit -1 + endif + if ($codecov == 1) then + echo "${0}: ERROR in arguments, must use --suite with --codecov" + exit -1 + endif if ("$compilers" =~ "*,*") then echo "${0}: ERROR in arguments, cannot set multiple compilers without --suite" exit -1 @@ -388,10 +420,7 @@ else endif cp -f ${ICE_SCRIPTS}/tests/report_results.csh ${tsdir} cp -f ${ICE_SCRIPTS}/tests/timeseries.csh ${tsdir} - - if ($report == 1) then - cp -f ${ICE_SCRIPTS}/tests/poll_queue.csh ${tsdir} - endif + cp -f ${ICE_SCRIPTS}/tests/poll_queue.csh ${tsdir} cat >! ${tsdir}/suite.run << EOF0 #!/bin/csh -f @@ -417,16 +446,29 @@ echo "#hash = ${hash}" >> results.log echo "#hshs = ${shhash}" >> results.log echo "#hshu = ${hashuser}" >> results.log echo "#hshd = ${hashdate}" >> results.log +echo "#suit = ${testsuite}" >> results.log echo "#date = ${cdate}" >> results.log echo "#time = ${ctime}" >> results.log echo "#mach = ${machine}" >> results.log echo "#user = ${user}" >> results.log echo "#vers = ${vers}" >> results.log echo "#------- " >> results.log +EOF0 + +cat >! ${tsdir}/report_codecov.csh << EOF0 +#!/bin/csh -f + +#setenv CODECOV_TOKEN "df12b574-8dce-439d-8d3b-ed7428d7598a" +setenv CODECOV_TOKEN "0b3feb13-0110-4618-821e-248c2f1f7cf3" +set report_name = "${shhash}:${branch}:${machine} ${testsuite}" + +set use_curl = 1 + EOF0 chmod +x ${tsdir}/suite.run chmod +x ${tsdir}/results.csh + chmod +x ${tsdir}/report_codecov.csh endif @@ -581,6 +623,8 @@ foreach compiler ( $ncompilers ) endif endif + set rundir = ${ICE_MACHINE_WKDIR}/${casename} + #------------------------------------------------------------ # Compute a default blocksize @@ -613,6 +657,7 @@ foreach compiler ( $ncompilers ) echo "ICE_CASEDIR = ${casedir}" echo "ICE_MACHINE = ${machine}" echo "ICE_COMPILER = ${compiler}" + echo "ICE_RUNDIR = ${rundir}" #------------------------------------------------------------ # Copy in and update icepack.settings and icepack_in files @@ -658,7 +703,7 @@ setenv ICE_CASEDIR ${casedir} setenv ICE_MACHINE ${machine} setenv ICE_COMPILER ${compiler} setenv ICE_MACHCOMP ${machcomp} -setenv ICE_RUNDIR ${ICE_MACHINE_WKDIR}/${casename} +setenv ICE_RUNDIR ${rundir} setenv ICE_GRID ${grid} setenv ICE_NXGLOB ${ICE_DECOMP_NXGLOB} setenv ICE_NTASKS ${task} @@ -673,6 +718,7 @@ setenv ICE_TESTNAME ${testname_noid} setenv ICE_BFBCOMP ${fbfbcomp} setenv ICE_ACCOUNT ${acct} setenv ICE_QUEUE ${queue} +setenv ICE_CODECOV ${codecovflag} EOF1 if (${sets} != "") then @@ -805,6 +851,13 @@ cat >> ${tsdir}/results.csh << EOF cat ${testname_base}/test_output >> results.log EOF +cat >> ${tsdir}/report_codecov.csh << EOF + +mkdir ${testname_base}/codecov +cp ${rundir}/compile/*.{gcno,gcda} ${testname_base}/codecov/ + +EOF + cat >> ${tsdir}/suite.run << EOF cd ${testname_base} ./icepack.build @@ -883,6 +936,16 @@ echo "\$failures of \$chkcnt tests FAILED" #echo " \$failbfbc of \$failures FAILED bfbcomp" #echo " \$failgen of \$failures FAILED generate" exit \$failures +EOF + +cat >> ${tsdir}/report_codecov.csh << EOF + +if ( \${use_curl} == 1 ) then + bash -c "bash <(curl -s https://codecov.io/bash) -n '\${report_name}'" +else + bash -c "bash <(wget -O - https://codecov.io/bash) -n '\${report_name}'" +endif + EOF # build and submit tests @@ -894,6 +957,11 @@ EOF ./results.csh ./report_results.csh endif + if ($codecov == 1) then + echo "Generating codecov reports" + ./poll_queue.csh + ./report_codecov.csh + endif cd ${ICE_SANDBOX} endif From 95abac8733782efa1cc0ce9c407bb77d38920f63 Mon Sep 17 00:00:00 2001 From: Anthony Craig Date: Fri, 10 Apr 2020 18:50:23 -0600 Subject: [PATCH 03/27] update codecov range --- .codecov.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.codecov.yml b/.codecov.yml index 0058c5f0b..cb580f0e6 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -1,2 +1,6 @@ # do not comment with code coverage metrics in github pull requests comment: false +coverage: + range: 20..100 + round: down + precision: 2 From 67c36b0b440a0a239b85553e7e7fce9c6a7e0524 Mon Sep 17 00:00:00 2001 From: Anthony Craig Date: Sat, 11 Apr 2020 11:20:04 -0600 Subject: [PATCH 04/27] update codecov.yml --- .codecov.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.codecov.yml b/.codecov.yml index cb580f0e6..aeb9654fb 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -1,6 +1,7 @@ # do not comment with code coverage metrics in github pull requests comment: false coverage: - range: 20..100 + range: "20...100" round: down precision: 2 + From 708c5158f1da79e4fe105bf9677d172b2fff6af0 Mon Sep 17 00:00:00 2001 From: Anthony Craig Date: Sat, 11 Apr 2020 11:32:49 -0600 Subject: [PATCH 05/27] update codecov.yml --- .codecov.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.codecov.yml b/.codecov.yml index aeb9654fb..9227e4c1f 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -1,5 +1,4 @@ -# do not comment with code coverage metrics in github pull requests -comment: false + coverage: range: "20...100" round: down From bdecfe04d018758d86e58a5cb4db16f6c444ba59 Mon Sep 17 00:00:00 2001 From: Anthony Craig Date: Sat, 11 Apr 2020 11:38:51 -0600 Subject: [PATCH 06/27] update icepack.setup --- icepack.setup | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/icepack.setup b/icepack.setup index 07c25ca56..739d6e53c 100755 --- a/icepack.setup +++ b/icepack.setup @@ -853,8 +853,8 @@ EOF cat >> ${tsdir}/report_codecov.csh << EOF -mkdir ${testname_base}/codecov -cp ${rundir}/compile/*.{gcno,gcda} ${testname_base}/codecov/ +mkdir ${testname_base}/codecov_output +cp ${rundir}/compile/*.{gcno,gcda} ${testname_base}/codecov_output/ EOF @@ -941,11 +941,14 @@ EOF cat >> ${tsdir}/report_codecov.csh << EOF if ( \${use_curl} == 1 ) then - bash -c "bash <(curl -s https://codecov.io/bash) -n '\${report_name}'" + bash -c "bash <(curl -s https://codecov.io/bash) -n '\${report_name}' -y .codecov.yml" else - bash -c "bash <(wget -O - https://codecov.io/bash) -n '\${report_name}'" + bash -c "bash <(wget -O - https://codecov.io/bash) -n '\${report_name}' -y .codecov.yml" endif +sleep 10 +rm -r -f ./*/codecov_output + EOF # build and submit tests From a55e3e0c57903d0b7b097daf6457ecac3db9f2f9 Mon Sep 17 00:00:00 2001 From: Anthony Craig Date: Sat, 11 Apr 2020 11:43:03 -0600 Subject: [PATCH 07/27] rename .codecov.yml codecov.yml --- codecov.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 codecov.yml diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 000000000..9227e4c1f --- /dev/null +++ b/codecov.yml @@ -0,0 +1,6 @@ + +coverage: + range: "20...100" + round: down + precision: 2 + From e96c04987bcd52fc1510a74e5f6f49afd762f6cc Mon Sep 17 00:00:00 2001 From: Anthony Craig Date: Sat, 11 Apr 2020 11:53:13 -0600 Subject: [PATCH 08/27] update icepack.setup --- icepack.setup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/icepack.setup b/icepack.setup index 739d6e53c..114cf30dc 100755 --- a/icepack.setup +++ b/icepack.setup @@ -941,9 +941,9 @@ EOF cat >> ${tsdir}/report_codecov.csh << EOF if ( \${use_curl} == 1 ) then - bash -c "bash <(curl -s https://codecov.io/bash) -n '\${report_name}' -y .codecov.yml" + bash -c "bash <(curl -s https://codecov.io/bash) -n '\${report_name}' " else - bash -c "bash <(wget -O - https://codecov.io/bash) -n '\${report_name}' -y .codecov.yml" + bash -c "bash <(wget -O - https://codecov.io/bash) -n '\${report_name}' " endif sleep 10 From 555c2f9bceaf3604cfc5cbf15ac2aad7d259f958 Mon Sep 17 00:00:00 2001 From: Anthony Craig Date: Sat, 11 Apr 2020 12:01:48 -0600 Subject: [PATCH 09/27] delete .codecov.yml --- .codecov.yml | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 .codecov.yml diff --git a/.codecov.yml b/.codecov.yml deleted file mode 100644 index 9227e4c1f..000000000 --- a/.codecov.yml +++ /dev/null @@ -1,6 +0,0 @@ - -coverage: - range: "20...100" - round: down - precision: 2 - From b0d93f6b4a5af2c0535d58e600650eefb54f4ea4 Mon Sep 17 00:00:00 2001 From: Anthony Craig Date: Sat, 11 Apr 2020 13:15:48 -0600 Subject: [PATCH 10/27] update icepack.setup --- icepack.setup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/icepack.setup b/icepack.setup index 114cf30dc..db2543b9d 100755 --- a/icepack.setup +++ b/icepack.setup @@ -941,9 +941,9 @@ EOF cat >> ${tsdir}/report_codecov.csh << EOF if ( \${use_curl} == 1 ) then - bash -c "bash <(curl -s https://codecov.io/bash) -n '\${report_name}' " + bash -c "bash <(curl -s https://codecov.io/bash) -n '\${report_name} -y ./codecov.yml' " else - bash -c "bash <(wget -O - https://codecov.io/bash) -n '\${report_name}' " + bash -c "bash <(wget -O - https://codecov.io/bash) -n '\${report_name} -y ./codecov.yml' " endif sleep 10 From aeeac6ae46606bf8bd3311dc376eec3ef4fa85a7 Mon Sep 17 00:00:00 2001 From: Anthony Craig Date: Sat, 11 Apr 2020 13:24:33 -0600 Subject: [PATCH 11/27] update codecov.yml --- codecov.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codecov.yml b/codecov.yml index 9227e4c1f..7f17a1f1c 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,6 +1,6 @@ coverage: - range: "20...100" + range: "40...70" round: down precision: 2 From a5815fece67a9885e9ae226f16b76857cf607bd9 Mon Sep 17 00:00:00 2001 From: Anthony Craig Date: Sat, 11 Apr 2020 13:42:21 -0600 Subject: [PATCH 12/27] update icepack.setup --- icepack.setup | 2 ++ 1 file changed, 2 insertions(+) diff --git a/icepack.setup b/icepack.setup index db2543b9d..071afbbcc 100755 --- a/icepack.setup +++ b/icepack.setup @@ -958,6 +958,8 @@ EOF echo "Reporting results" ./poll_queue.csh ./results.csh + cp report_results.csh ../ + cd .. ./report_results.csh endif if ($codecov == 1) then From f5221bb97b8ca976d8eb6748a935d3c38d2c55fd Mon Sep 17 00:00:00 2001 From: Anthony Craig Date: Sat, 11 Apr 2020 13:55:08 -0600 Subject: [PATCH 13/27] update icepack.setup --- icepack.setup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/icepack.setup b/icepack.setup index 071afbbcc..f4f482a85 100755 --- a/icepack.setup +++ b/icepack.setup @@ -958,13 +958,13 @@ EOF echo "Reporting results" ./poll_queue.csh ./results.csh - cp report_results.csh ../ - cd .. ./report_results.csh endif if ($codecov == 1) then echo "Generating codecov reports" ./poll_queue.csh + cp ./report_codecov.csh ../ + cd ../ ./report_codecov.csh endif cd ${ICE_SANDBOX} From 48ac1a4bbe51acdc100bd100a6bfb219860243b4 Mon Sep 17 00:00:00 2001 From: Anthony Craig Date: Sat, 11 Apr 2020 14:01:50 -0600 Subject: [PATCH 14/27] update icepack.setup --- icepack.setup | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/icepack.setup b/icepack.setup index f4f482a85..99a6b8dbf 100755 --- a/icepack.setup +++ b/icepack.setup @@ -940,6 +940,8 @@ EOF cat >> ${tsdir}/report_codecov.csh << EOF +cd ../ + if ( \${use_curl} == 1 ) then bash -c "bash <(curl -s https://codecov.io/bash) -n '\${report_name} -y ./codecov.yml' " else @@ -947,7 +949,7 @@ else endif sleep 10 -rm -r -f ./*/codecov_output +rm -r -f ./*/*/codecov_output EOF @@ -963,8 +965,6 @@ EOF if ($codecov == 1) then echo "Generating codecov reports" ./poll_queue.csh - cp ./report_codecov.csh ../ - cd ../ ./report_codecov.csh endif cd ${ICE_SANDBOX} From bb72c2c5cfdbe31b115a2cf360f669ead4eb65b9 Mon Sep 17 00:00:00 2001 From: Anthony Craig Date: Sat, 11 Apr 2020 14:06:21 -0600 Subject: [PATCH 15/27] update icepack.setup --- icepack.setup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/icepack.setup b/icepack.setup index 99a6b8dbf..5d0f62d55 100755 --- a/icepack.setup +++ b/icepack.setup @@ -940,7 +940,7 @@ EOF cat >> ${tsdir}/report_codecov.csh << EOF -cd ../ +#cd ../ if ( \${use_curl} == 1 ) then bash -c "bash <(curl -s https://codecov.io/bash) -n '\${report_name} -y ./codecov.yml' " @@ -949,7 +949,7 @@ else endif sleep 10 -rm -r -f ./*/*/codecov_output +rm -r -f ./*/codecov_output EOF From 527637e3658be29a87fb66f9da46820e79422fb6 Mon Sep 17 00:00:00 2001 From: Anthony Craig Date: Sat, 11 Apr 2020 14:13:53 -0600 Subject: [PATCH 16/27] update icepack.setup and switch back to .codecov.yml --- codecov.yml => .codecov.yml | 4 ++-- icepack.setup | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) rename codecov.yml => .codecov.yml (53%) diff --git a/codecov.yml b/.codecov.yml similarity index 53% rename from codecov.yml rename to .codecov.yml index 7f17a1f1c..e7f6a6e2f 100644 --- a/codecov.yml +++ b/.codecov.yml @@ -1,6 +1,6 @@ - coverage: - range: "40...70" + range: "20...100" round: down precision: 2 +comment: false diff --git a/icepack.setup b/icepack.setup index 5d0f62d55..ec994d63c 100755 --- a/icepack.setup +++ b/icepack.setup @@ -943,9 +943,9 @@ cat >> ${tsdir}/report_codecov.csh << EOF #cd ../ if ( \${use_curl} == 1 ) then - bash -c "bash <(curl -s https://codecov.io/bash) -n '\${report_name} -y ./codecov.yml' " + bash -c "bash <(curl -s https://codecov.io/bash) -n '\${report_name} -y ./.codecov.yml' " else - bash -c "bash <(wget -O - https://codecov.io/bash) -n '\${report_name} -y ./codecov.yml' " + bash -c "bash <(wget -O - https://codecov.io/bash) -n '\${report_name} -y ./.codecov.yml' " endif sleep 10 From ad96e0825ebd9f0baf0f2de99ed277db88837113 Mon Sep 17 00:00:00 2001 From: Anthony Craig Date: Sat, 11 Apr 2020 14:20:53 -0600 Subject: [PATCH 17/27] update icepack.setup --- icepack.setup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/icepack.setup b/icepack.setup index ec994d63c..e0094115b 100755 --- a/icepack.setup +++ b/icepack.setup @@ -943,9 +943,9 @@ cat >> ${tsdir}/report_codecov.csh << EOF #cd ../ if ( \${use_curl} == 1 ) then - bash -c "bash <(curl -s https://codecov.io/bash) -n '\${report_name} -y ./.codecov.yml' " + bash -c "bash <(curl -s https://codecov.io/bash) -n '\${report_name}' -y ./.codecov.yml " else - bash -c "bash <(wget -O - https://codecov.io/bash) -n '\${report_name} -y ./.codecov.yml' " + bash -c "bash <(wget -O - https://codecov.io/bash) -n '\${report_name}' -y ./.codecov.yml " endif sleep 10 From d90b9ac16bd4327ab30982fd662da478cd7cfbd2 Mon Sep 17 00:00:00 2001 From: Anthony Craig Date: Sat, 11 Apr 2020 14:24:17 -0600 Subject: [PATCH 18/27] move back to codecov.yml --- .codecov.yml => codecov.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .codecov.yml => codecov.yml (100%) diff --git a/.codecov.yml b/codecov.yml similarity index 100% rename from .codecov.yml rename to codecov.yml From 017ca2954dd515370eb51d08786b4e0f4c005c1a Mon Sep 17 00:00:00 2001 From: Anthony Craig Date: Sat, 11 Apr 2020 14:32:56 -0600 Subject: [PATCH 19/27] update icepack.setup --- icepack.setup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/icepack.setup b/icepack.setup index e0094115b..23a780f88 100755 --- a/icepack.setup +++ b/icepack.setup @@ -943,9 +943,9 @@ cat >> ${tsdir}/report_codecov.csh << EOF #cd ../ if ( \${use_curl} == 1 ) then - bash -c "bash <(curl -s https://codecov.io/bash) -n '\${report_name}' -y ./.codecov.yml " + bash -c "bash <(curl -s https://codecov.io/bash) -n '\${report_name}' -y ./codecov.yml " else - bash -c "bash <(wget -O - https://codecov.io/bash) -n '\${report_name}' -y ./.codecov.yml " + bash -c "bash <(wget -O - https://codecov.io/bash) -n '\${report_name}' -y ./codecov.yml " endif sleep 10 From 3db3f5dd8b4500fdfb06cf402cfb4bb6f61cfd01 Mon Sep 17 00:00:00 2001 From: apcraig Date: Mon, 13 Apr 2020 17:29:18 +0000 Subject: [PATCH 20/27] update code coverage implementation --- README.md | 1 + icepack.setup | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 16ba9ac7e..4a37dc3e4 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ [![Build Status](https://travis-ci.org/CICE-Consortium/Icepack.svg?branch=master)](https://travis-ci.org/CICE-Consortium/Icepack) [![Documentation Status](https://readthedocs.org/projects/cice-consortium-icepack/badge/?version=master)](http://cice-consortium-icepack.readthedocs.io/en/master/?badge=master) +[![codecov](https://codecov.io/gh/CICE-Consortium/Icepack/branch/master/graph/badge.svg)](https://codecov.io/gh/CICE-Consortium/Icepack) ## The Icepack sea-ice column model This repository contains files for Icepack, the column physics of the sea ice model [CICE][cice]. Icepack is maintained by the CICE Consortium. For testing purposes and guidance for including Icepack in other sea ice host models, this repository also includes a driver and basic test suite. diff --git a/icepack.setup b/icepack.setup index 23a780f88..c5f053858 100755 --- a/icepack.setup +++ b/icepack.setup @@ -458,8 +458,8 @@ EOF0 cat >! ${tsdir}/report_codecov.csh << EOF0 #!/bin/csh -f -#setenv CODECOV_TOKEN "df12b574-8dce-439d-8d3b-ed7428d7598a" -setenv CODECOV_TOKEN "0b3feb13-0110-4618-821e-248c2f1f7cf3" +#setenv CODECOV_TOKEN "df12b574-8dce-439d-8d3b-ed7428d7598a" # consortium icepack +setenv CODECOV_TOKEN "0b3feb13-0110-4618-821e-248c2f1f7cf3" # apcraig icepack set report_name = "${shhash}:${branch}:${machine} ${testsuite}" set use_curl = 1 From da10137d5577217c98f006f4946d40069dbc04dc Mon Sep 17 00:00:00 2001 From: Anthony Craig Date: Mon, 13 Apr 2020 12:01:29 -0600 Subject: [PATCH 21/27] code coverage implementation --- .travis.yml | 1 - README.md | 1 + configuration/scripts/icepack.settings | 1 + .../scripts/machines/Macros.conrad_gnu | 20 ++++++++++--- .../scripts/machines/Macros.gaffney_gnu | 20 ++++++++++--- .../scripts/machines/Macros.gordon_gnu | 20 ++++++++++--- .../scripts/machines/Macros.high_Sierra_gnu | 2 +- .../scripts/machines/Macros.izumi_gnu | 20 ++++++++++--- .../scripts/machines/Macros.koehr_gnu | 22 ++++++++++---- .../scripts/machines/Macros.onyx_gnu | 20 ++++++++++--- .../scripts/machines/Macros.travisCI_gnu | 24 +++++++++++---- .../scripts/tests/report_results.csh | 2 ++ doc/source/developer_guide/dg_scripts.rst | 30 ++++++++++++++----- doc/source/user_guide/ug_testing.rst | 21 +++++++++++++ icepack.setup | 10 ++----- 15 files changed, 166 insertions(+), 48 deletions(-) diff --git a/.travis.yml b/.travis.yml index ca4b34b5b..85cc22e4b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,6 @@ install: #- "lftp ftp://anonymous:travis@travis-ci.org@ftp.cgd.ucar.edu #-e 'mirror /archive/Model-Data/CICE/ ~/ICEPACK_INPUTDATA; quit'" - script: - "./icepack.setup --suite travis_suite --testid travisCItest --mach travisCI --env gnu && diff --git a/README.md b/README.md index 16ba9ac7e..cf2f047cb 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ [![Build Status](https://travis-ci.org/CICE-Consortium/Icepack.svg?branch=master)](https://travis-ci.org/CICE-Consortium/Icepack) [![Documentation Status](https://readthedocs.org/projects/cice-consortium-icepack/badge/?version=master)](http://cice-consortium-icepack.readthedocs.io/en/master/?badge=master) +[![codecov.io](http://codecov.io/github/CICE-Consortium/Icepack/coverage.svg?branch=master)](http://codecov.io/github/CICE-Consortium/Icepack?branch=master) ## The Icepack sea-ice column model This repository contains files for Icepack, the column physics of the sea ice model [CICE][cice]. Icepack is maintained by the CICE Consortium. For testing purposes and guidance for including Icepack in other sea ice host models, this repository also includes a driver and basic test suite. diff --git a/configuration/scripts/icepack.settings b/configuration/scripts/icepack.settings index 8860c103d..295cacfa2 100755 --- a/configuration/scripts/icepack.settings +++ b/configuration/scripts/icepack.settings @@ -77,5 +77,6 @@ setenv TRFED 0 # number of dissolved iron tracers ### Specialty code setenv ICE_BLDDEBUG false # build debug flags +setenv ICE_CODECOV false # code coverage flag diff --git a/configuration/scripts/machines/Macros.conrad_gnu b/configuration/scripts/machines/Macros.conrad_gnu index 2a62f7de5..dba7798f6 100644 --- a/configuration/scripts/machines/Macros.conrad_gnu +++ b/configuration/scripts/machines/Macros.conrad_gnu @@ -4,7 +4,7 @@ CPP := ftn -E CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} -CFLAGS := -c -O2 -ffloat-store -march=native +CFLAGS := -c -ffloat-store -march=native FIXEDFLAGS := -ffixed-line-length-132 FREEFLAGS := -ffree-form @@ -12,9 +12,21 @@ FFLAGS := -ffloat-store -fconvert=swap -fbacktrace -march=native -ffree-line FFLAGS_NOOPT:= -O0 ifeq ($(ICE_BLDDEBUG), true) - FFLAGS += -O0 -g -Wuninitialized -fbounds-check -ffpe-trap=invalid,zero,overflow,underflow -else - FFLAGS += -O2 + FFLAGS += -O0 -g -Wuninitialized -fbounds-check -ffpe-trap=invalid,zero,overflow,underflow + CFLAGS += -O0 +endif + +ifeq ($(ICE_CODECOV), true) + FFLAGS += -O0 -g -fprofile-arcs -ftest-coverage + CFLAGS += -O0 -g -coverage + LDFLAGS += -g -ftest-coverage -fprofile-arcs +endif + +ifneq ($(ICE_BLDDEBUG), true) +ifneq ($(ICE_CODECOV), true) + FFLAGS += -O2 + CFLAGS += -O2 +endif endif SCC := cc diff --git a/configuration/scripts/machines/Macros.gaffney_gnu b/configuration/scripts/machines/Macros.gaffney_gnu index 0047f951d..febf07bd4 100644 --- a/configuration/scripts/machines/Macros.gaffney_gnu +++ b/configuration/scripts/machines/Macros.gaffney_gnu @@ -4,7 +4,7 @@ CPP := ftn -E CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} -CFLAGS := -c -O2 +CFLAGS := -c FIXEDFLAGS := -ffixed-line-length-132 FREEFLAGS := -ffree-form @@ -12,9 +12,21 @@ FFLAGS := -fconvert=big-endian -fbacktrace -ffree-line-length-none FFLAGS_NOOPT:= -O0 ifeq ($(ICE_BLDDEBUG), true) - FFLAGS += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow -else - FFLAGS += -O2 + FFLAGS += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow + CFLAGS += -O0 +endif + +ifeq ($(ICE_CODECOV), true) + FFLAGS += -O0 -g -fprofile-arcs -ftest-coverage + CFLAGS += -O0 -g -coverage + LDFLAGS += -g -ftest-coverage -fprofile-arcs +endif + +ifneq ($(ICE_BLDDEBUG), true) +ifneq ($(ICE_CODECOV), true) + FFLAGS += -O2 + CFLAGS += -O2 +endif endif SCC := gcc diff --git a/configuration/scripts/machines/Macros.gordon_gnu b/configuration/scripts/machines/Macros.gordon_gnu index 1fcf530da..91b19cbfa 100644 --- a/configuration/scripts/machines/Macros.gordon_gnu +++ b/configuration/scripts/machines/Macros.gordon_gnu @@ -4,7 +4,7 @@ CPP := ftn -E CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} -CFLAGS := -c -O2 -ffloat-store -march=native +CFLAGS := -c -ffloat-store -march=native FIXEDFLAGS := -ffixed-line-length-132 FREEFLAGS := -ffree-form @@ -12,9 +12,21 @@ FFLAGS := -ffloat-store -fconvert=swap -fbacktrace -march=native -ffree-line FFLAGS_NOOPT:= -O0 ifeq ($(ICE_BLDDEBUG), true) - FFLAGS += -O0 -g -Wuninitialized -fbounds-check -ffpe-trap=invalid,zero,overflow,underflow -else - FFLAGS += -O2 + FFLAGS += -O0 -g -Wuninitialized -fbounds-check -ffpe-trap=invalid,zero,overflow,underflow + CFLAGS += -O0 +endif + +ifeq ($(ICE_CODECOV), true) + FFLAGS += -O0 -g -fprofile-arcs -ftest-coverage + CFLAGS += -O0 -g -coverage + LDFLAGS += -g -ftest-coverage -fprofile-arcs +endif + +ifneq ($(ICE_BLDDEBUG), true) +ifneq ($(ICE_CODECOV), true) + FFLAGS += -O2 + CFLAGS += -O2 +endif endif SCC := cc diff --git a/configuration/scripts/machines/Macros.high_Sierra_gnu b/configuration/scripts/machines/Macros.high_Sierra_gnu index df2911f66..d2a0b257e 100644 --- a/configuration/scripts/machines/Macros.high_Sierra_gnu +++ b/configuration/scripts/machines/Macros.high_Sierra_gnu @@ -1,5 +1,5 @@ #============================================================================== -# Makefile macros for Travis-CI - GCC and openmpi compilers +# Makefile macros for high_Sierra - GCC and openmpi compilers #============================================================================== CPP := cpp diff --git a/configuration/scripts/machines/Macros.izumi_gnu b/configuration/scripts/machines/Macros.izumi_gnu index d42cce39e..48669c1b6 100644 --- a/configuration/scripts/machines/Macros.izumi_gnu +++ b/configuration/scripts/machines/Macros.izumi_gnu @@ -4,7 +4,7 @@ CPP := ftn -E CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} -CFLAGS := -c -O2 +CFLAGS := -c FIXEDFLAGS := -ffixed-line-length-132 FREEFLAGS := -ffree-form @@ -12,9 +12,21 @@ FFLAGS := -fconvert=big-endian -fbacktrace -ffree-line-length-none FFLAGS_NOOPT:= -O0 ifeq ($(ICE_BLDDEBUG), true) - FFLAGS += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow -else - FFLAGS += -O2 + FFLAGS += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow + CFLAGS += -O0 +endif + +ifeq ($(ICE_CODECOV), true) + FFLAGS += -O0 -g -fprofile-arcs -ftest-coverage + CFLAGS += -O0 -g -coverage + LDFLAGS += -g -ftest-coverage -fprofile-arcs +endif + +ifneq ($(ICE_BLDDEBUG), true) +ifneq ($(ICE_CODECOV), true) + FFLAGS += -O2 + CFLAGS += -O2 +endif endif SCC := gcc diff --git a/configuration/scripts/machines/Macros.koehr_gnu b/configuration/scripts/machines/Macros.koehr_gnu index 97950c249..b34dfa3ec 100644 --- a/configuration/scripts/machines/Macros.koehr_gnu +++ b/configuration/scripts/machines/Macros.koehr_gnu @@ -1,10 +1,10 @@ #============================================================================== -# Macros file for NAVYDSRC koehr, intel compiler +# Macros file for NAVYDSRC koehr, gnu compiler #============================================================================== CPP := ftn -E CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} -CFLAGS := -c -O2 +CFLAGS := -c FIXEDFLAGS := -ffixed-line-length-132 FREEFLAGS := -ffree-form @@ -12,9 +12,21 @@ FFLAGS := -fconvert=big-endian -fbacktrace -ffree-line-length-none FFLAGS_NOOPT:= -O0 ifeq ($(ICE_BLDDEBUG), true) - FFLAGS += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow -else - FFLAGS += -O2 + FFLAGS += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow + CFLAGS += -O0 +endif + +ifeq ($(ICE_CODECOV), true) + FFLAGS += -O0 -g -fprofile-arcs -ftest-coverage + CFLAGS += -O0 -g -coverage + LDFLAGS += -g -ftest-coverage -fprofile-arcs +endif + +ifneq ($(ICE_BLDDEBUG), true) +ifneq ($(ICE_CODECOV), true) + FFLAGS += -O2 + CFLAGS += -O2 +endif endif SCC := gcc diff --git a/configuration/scripts/machines/Macros.onyx_gnu b/configuration/scripts/machines/Macros.onyx_gnu index d22562760..ba18e077e 100644 --- a/configuration/scripts/machines/Macros.onyx_gnu +++ b/configuration/scripts/machines/Macros.onyx_gnu @@ -4,7 +4,7 @@ CPP := ftn -E CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} -CFLAGS := -c -O2 -ffloat-store -march=native +CFLAGS := -c -ffloat-store -march=native FIXEDFLAGS := -ffixed-line-length-132 FREEFLAGS := -ffree-form @@ -12,9 +12,21 @@ FFLAGS := -ffloat-store -fconvert=swap -fbacktrace -march=native -ffree-line FFLAGS_NOOPT:= -O0 ifeq ($(ICE_BLDDEBUG), true) - FFLAGS += -O0 -g -Wuninitialized -fbounds-check -ffpe-trap=invalid,zero,overflow,underflow -else - FFLAGS += -O2 + FFLAGS += -O0 -g -Wuninitialized -fbounds-check -ffpe-trap=invalid,zero,overflow,underflow + CFLAGS += -O0 +endif + +ifeq ($(ICE_CODECOV), true) + FFLAGS += -O0 -g -fprofile-arcs -ftest-coverage + CFLAGS += -O0 -g -coverage + LDFLAGS += -g -ftest-coverage -fprofile-arcs +endif + +ifneq ($(ICE_BLDDEBUG), true) +ifneq ($(ICE_CODECOV), true) + FFLAGS += -O2 + CFLAGS += -O2 +endif endif SCC := cc diff --git a/configuration/scripts/machines/Macros.travisCI_gnu b/configuration/scripts/machines/Macros.travisCI_gnu index 1ef6b44a2..6e20c2beb 100644 --- a/configuration/scripts/machines/Macros.travisCI_gnu +++ b/configuration/scripts/machines/Macros.travisCI_gnu @@ -4,16 +4,28 @@ CPP := /usr/bin/cpp CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} -CFLAGS := -c -O2 -fp-model precise -xHost +CFLAGS := -c -fp-model precise -xHost FIXEDFLAGS := -132 -FFLAGS := -O2 -ffree-line-length-none -fconvert=big-endian -finit-real=nan +FFLAGS := -ffree-line-length-none -fconvert=big-endian -finit-real=nan FFLAGS_NOOPT:= -O0 ifeq ($(ICE_BLDDEBUG), true) - FFLAGS += -O0 -g -Wextra -fbacktrace -fbounds-check -ffpe-trap=zero,overflow -else - FFLAGS += -O2 + FFLAGS += -O0 -g -Wextra -fbacktrace -fbounds-check -ffpe-trap=zero,overflow + CFLAGS += -O0 +endif + +ifeq ($(ICE_CODECOV), true) + FFLAGS += -O0 -g -fprofile-arcs -ftest-coverage + CFLAGS += -O0 -g -coverage + LDFLAGS += -g -ftest-coverage -fprofile-arcs +endif + +ifneq ($(ICE_BLDDEBUG), true) +ifneq ($(ICE_CODECOV), true) + FFLAGS += -O2 + CFLAGS += -O2 +endif endif SCC := gcc @@ -29,7 +41,7 @@ ifeq ($(ICE_IOTYPE), netcdf) INCLDIR := $(INCLDIR) -I$(NETCDF_PATH)/include LIB_NETCDF := $(NETCDF_PATH)/lib LIB_PNETCDF := - SLIBS := -L$(LIB_NETCDF) -lnetcdf + SLIBS := -L$(LIB_NETCDF) -lnetcdf else SLIBS := endif diff --git a/configuration/scripts/tests/report_results.csh b/configuration/scripts/tests/report_results.csh index a52294a8a..141bc45c2 100755 --- a/configuration/scripts/tests/report_results.csh +++ b/configuration/scripts/tests/report_results.csh @@ -39,6 +39,7 @@ set hash = `grep "#hash = " results.log | cut -c 9-` set shhash = `grep "#hshs = " results.log | cut -c 9-` set hashuser = `grep "#hshu = " results.log | cut -c 9-` set hashdate = `grep "#hshd = " results.log | cut -c 9-` +set testsuites = `grep "#suit = " results.log | cut -c 9-` set cdat = `grep "#date = " results.log | cut -c 9-` set ctim = `grep "#time = " results.log | cut -c 9-` set user = `grep "#user = " results.log | cut -c 9-` @@ -56,6 +57,7 @@ set compilers = `grep -v "#" results.log | grep ${mach}_ | cut -d "_" -f 2 | sor #echo "debug ${shhash}" #echo "debug ${hashuser}" #echo "debug ${hashdate}" +#echo "debug ${testsuites}" #echo "debug ${cdat}" #echo "debug ${ctim}" #echo "debug ${user}" diff --git a/doc/source/developer_guide/dg_scripts.rst b/doc/source/developer_guide/dg_scripts.rst index e76c8cac6..fe6b6d613 100755 --- a/doc/source/developer_guide/dg_scripts.rst +++ b/doc/source/developer_guide/dg_scripts.rst @@ -91,14 +91,28 @@ Test scripts ------------- Under **configuration/scripts/tests** are several files including the scripts to -setup the smoke and restart tests (**test_smoke.script**, **test_restart.script*). -A baseline test script (**baseline.script**) is also there to setup the regression +setup the various tests, such as smoke and restart tests (**test_smoke.script**, +**test_restart.script**). +and the files that describe which options files are needed for each test +(ie. **test_smoke.files**, **test_restart.files**). +A baseline test script (**baseline.script**) is also there to setup the general regression and comparison testing. That directory also contains the preset test suites (ie. **base_suite.ts**) and a file that supports post-processing on the model -output (**timeseries.csh**). +output (**timeseries.csh**). There is also a script **report_results.csh** that +pushes results from test suites back to the CICE-Consortium test results wiki page. + +To add a new test (for example newtest), several files may be needed, + +- **configuration/scripts/tests/test_newtest.script** defines how to run the test. This chunk + of script will be incorporated into the case test script +- **configuration/scripts/tests/test_newtest.files** list the set of options files found in + **configuration/scripts/options/** needed to + run this test. Those files will be copied into the test directory when the test is invoked + so they are available for the **test_newtest.script** to use. +- some new files may be needed in **configuration/scripts/options/**. These could be + relatively generic **set_nml** or **set_env** files, or they could be test specific files + typically carrying a prefix of **test_nml**. + +Generating a new test, particularly the **test_newtest.script** usually takes some iteration +before it's working properly. -To add a new test, a file associated with that test will need to be added to the -**configuration/scripts/tests** directory similar to **test_smoke.script** -and **test_restart.script**. In addition, some new options files in -**configuration/scripts/options** may need to be added similar to **test_nml.restart1**, -**test_nml.restart2**, and **set_nml.restart**. diff --git a/doc/source/user_guide/ug_testing.rst b/doc/source/user_guide/ug_testing.rst index 34a34086c..cdde0ee6e 100755 --- a/doc/source/user_guide/ug_testing.rst +++ b/doc/source/user_guide/ug_testing.rst @@ -304,6 +304,9 @@ following options are valid for suites, ``--report`` This is only used by ``--suite`` and when set, invokes a script that sends the test results to the results page when all tests are complete. Please see :ref:`testreporting` for more information. +``--codecov`` + When invoked, code coverage diagnostics are run. This includes modifications to the build, including reduced optimization. The results will be uploaded to the **codecov.io** website automaticaly via the **report_codecov.csh** script. Please see :ref:`codecoverage` for more information. + Please see :ref:`case_options` and :ref:`indtests` for more details about how these options are used. @@ -546,3 +549,21 @@ This plotting script can be used to plot the following variables: - congelation (m) - snow-ice (m) - initial energy change (:math:`W/m^2`) + +.. _codecoverage: + +Code Coverage Testing +--------------- + +The ``--codecov`` feature in **icepack.setup** provides a method to diagnose code coverage. +This argument turns on special compiler flags including reduced optimization. +This option is currently only available with the gnu compiler and on a few systems. +To use, submit a full test suite with the gnu compiler and the ``--codecov`` argument. +The test suite will run and a report will be generated and uploaded to +the `codecov.io site `_ by the **report_codecov.csh** script. +This is just a diagnostic test +and should not be treated as a test of the model due to changes in the compiler +settings. A sample job submission would look like :: + +$ ./icepack.setup -m conrad -e gnu --suite base_suite,travis_suite,quick_suite --testid cc01 --codecov + diff --git a/icepack.setup b/icepack.setup index 23a780f88..9ef52873b 100755 --- a/icepack.setup +++ b/icepack.setup @@ -82,7 +82,7 @@ SYNOPSIS --suite SUITE[,SUITE2] -m MACH --testid ID [-e ENV1,ENV2][--acct ACCT][--bdir DIR][--bgen DIR] - [--bcmp DIR][--tdir PATH][--report][--codecov] + [--bcmp DIR][--tdir PATH][--report || --codecov] DESCRIPTION --help, -h : help @@ -458,8 +458,8 @@ EOF0 cat >! ${tsdir}/report_codecov.csh << EOF0 #!/bin/csh -f -#setenv CODECOV_TOKEN "df12b574-8dce-439d-8d3b-ed7428d7598a" -setenv CODECOV_TOKEN "0b3feb13-0110-4618-821e-248c2f1f7cf3" +#setenv CODECOV_TOKEN "df12b574-8dce-439d-8d3b-ed7428d7598a" # cice-consortium icepack +setenv CODECOV_TOKEN "0b3feb13-0110-4618-821e-248c2f1f7cf3" ! apcraig icepack set report_name = "${shhash}:${branch}:${machine} ${testsuite}" set use_curl = 1 @@ -852,7 +852,6 @@ cat ${testname_base}/test_output >> results.log EOF cat >> ${tsdir}/report_codecov.csh << EOF - mkdir ${testname_base}/codecov_output cp ${rundir}/compile/*.{gcno,gcda} ${testname_base}/codecov_output/ @@ -939,9 +938,6 @@ exit \$failures EOF cat >> ${tsdir}/report_codecov.csh << EOF - -#cd ../ - if ( \${use_curl} == 1 ) then bash -c "bash <(curl -s https://codecov.io/bash) -n '\${report_name}' -y ./codecov.yml " else From f8c8159356b60d952e25b5092894eb5cd6f600f0 Mon Sep 17 00:00:00 2001 From: apcraig Date: Mon, 13 Apr 2020 18:28:48 +0000 Subject: [PATCH 22/27] update icepack.setup --- icepack.setup | 2 ++ 1 file changed, 2 insertions(+) diff --git a/icepack.setup b/icepack.setup index fe899c04f..3d5a25df0 100755 --- a/icepack.setup +++ b/icepack.setup @@ -938,6 +938,8 @@ exit \$failures EOF cat >> ${tsdir}/report_codecov.csh << EOF +source ${ICE_SCRIPTS}/machines/env.${machcomp} + if ( \${use_curl} == 1 ) then bash -c "bash <(curl -s https://codecov.io/bash) -n '\${report_name}' -y ./codecov.yml " else From 5b68a9cf3082923dd006e26102c0d564b2b602ac Mon Sep 17 00:00:00 2001 From: apcraig Date: Mon, 13 Apr 2020 20:40:01 +0000 Subject: [PATCH 23/27] update icepack token --- icepack.setup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/icepack.setup b/icepack.setup index 3d5a25df0..3921d26b1 100755 --- a/icepack.setup +++ b/icepack.setup @@ -458,8 +458,8 @@ EOF0 cat >! ${tsdir}/report_codecov.csh << EOF0 #!/bin/csh -f -#setenv CODECOV_TOKEN "df12b574-8dce-439d-8d3b-ed7428d7598a" # consortium icepack -setenv CODECOV_TOKEN "0b3feb13-0110-4618-821e-248c2f1f7cf3" # apcraig icepack +setenv CODECOV_TOKEN "df12b574-8dce-439d-8d3b-ed7428d7598a" # consortium icepack +#setenv CODECOV_TOKEN "0b3feb13-0110-4618-821e-248c2f1f7cf3" # apcraig icepack set report_name = "${shhash}:${branch}:${machine} ${testsuite}" set use_curl = 1 From b94e2b50266c063a6d08451c44fd11a7860832c4 Mon Sep 17 00:00:00 2001 From: apcraig Date: Tue, 14 Apr 2020 17:57:56 +0000 Subject: [PATCH 24/27] update documentation --- doc/source/user_guide/ug_testing.rst | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/doc/source/user_guide/ug_testing.rst b/doc/source/user_guide/ug_testing.rst index cdde0ee6e..332b8b56f 100755 --- a/doc/source/user_guide/ug_testing.rst +++ b/doc/source/user_guide/ug_testing.rst @@ -305,7 +305,7 @@ following options are valid for suites, This is only used by ``--suite`` and when set, invokes a script that sends the test results to the results page when all tests are complete. Please see :ref:`testreporting` for more information. ``--codecov`` - When invoked, code coverage diagnostics are run. This includes modifications to the build, including reduced optimization. The results will be uploaded to the **codecov.io** website automaticaly via the **report_codecov.csh** script. Please see :ref:`codecoverage` for more information. + When invoked, code coverage diagnostics are generated. This will modify the build and reduce optimization. The results will be uploaded to the **codecov.io** website via the **report_codecov.csh** script. General use is not recommended, this is mainly used as a diagnostic to periodically assess test coverage. Please see :ref:`codecoverage` for more information. Please see :ref:`case_options` and :ref:`indtests` for more details about how these options are used. @@ -553,17 +553,21 @@ This plotting script can be used to plot the following variables: .. _codecoverage: Code Coverage Testing ---------------- +------------------------ The ``--codecov`` feature in **icepack.setup** provides a method to diagnose code coverage. -This argument turns on special compiler flags including reduced optimization. +This argument turns on special compiler flags including reduced optimization and then +invokes the gcov tool. This option is currently only available with the gnu compiler and on a few systems. -To use, submit a full test suite with the gnu compiler and the ``--codecov`` argument. -The test suite will run and a report will be generated and uploaded to -the `codecov.io site `_ by the **report_codecov.csh** script. -This is just a diagnostic test -and should not be treated as a test of the model due to changes in the compiler -settings. A sample job submission would look like :: +To use, submit a full test suite using a version of Icepack on the Consortium master +and the gnu compiler with the ``--codecov`` argument to **icepack.setup**. +The test suite will run and then a report will be generated and uploaded to +the `codecov.io site `_ by the +**report_codecov.csh** script. +This is a special diagnostic test and does not constitute proper model testing. +General use is not recommended, this is mainly used as a diagnostic to periodically +assess test coverage. In addition, the interaction with codecov.io is not always robust. +A sample job submission would look like :: $ ./icepack.setup -m conrad -e gnu --suite base_suite,travis_suite,quick_suite --testid cc01 --codecov From 094400332afe10f3820c91aa72f11762ffe73c41 Mon Sep 17 00:00:00 2001 From: apcraig Date: Tue, 14 Apr 2020 19:00:15 +0000 Subject: [PATCH 25/27] Update documenation --- doc/source/user_guide/ug_testing.rst | 54 +++++++++++++++++----------- 1 file changed, 33 insertions(+), 21 deletions(-) diff --git a/doc/source/user_guide/ug_testing.rst b/doc/source/user_guide/ug_testing.rst index 332b8b56f..5deb40042 100755 --- a/doc/source/user_guide/ug_testing.rst +++ b/doc/source/user_guide/ug_testing.rst @@ -496,6 +496,39 @@ The reporting can also be automated in a test suite by adding ``--report`` to `` With ``--report``, the suite will create all the tests, build and submit them, wait for all runs to be complete, and run the results and report_results scripts. +.. _codecoverage: + +Code Coverage Testing +------------------------ + +The ``--codecov`` feature in **icepack.setup** provides a method to diagnose code coverage. +This argument turns on special compiler flags including reduced optimization and then +invokes the gcov tool. +This option is currently only available with the gnu compiler and on a few systems. +To use, submit a full test suite using a version of Icepack on the Consortium master +and the gnu compiler with the ``--codecov`` argument. +The test suite will run and then a report will be generated and uploaded to +the `codecov.io site `_ by the +**report_codecov.csh** script. + +This is a special diagnostic test and does not constitute proper model testing. +General use is not recommended, this is mainly used as a diagnostic to periodically +assess test coverage. The interaction with codecov.io is not always robust and +can be tricky to manage. Some constraints are that the output generated at runtime +is copied into the directory where compilation took place. That means each +test should be compiled separately. Tests that invoke multiple runs +(such as exact restart) will only save coverage information +for the last run, so some coverage information may be lost. The gcov tool can +be a little slow to run on large test suites, and the codecov.io bash uploader +(that runs gcov and uploads the data to codecov.io) is constantly evolving. +Finally, gcov requires that the diagnostic output be copied into the git sandbox for +analysis. These constraints are handled by the current scripts, but may change +in the future. + +A sample job submission would look like :: + +$ ./icepack.setup -m conrad -e gnu --suite base_suite,travis_suite,quick_suite --testid cc01 --codecov + .. _testplotting: Test Plotting @@ -550,24 +583,3 @@ This plotting script can be used to plot the following variables: - snow-ice (m) - initial energy change (:math:`W/m^2`) -.. _codecoverage: - -Code Coverage Testing ------------------------- - -The ``--codecov`` feature in **icepack.setup** provides a method to diagnose code coverage. -This argument turns on special compiler flags including reduced optimization and then -invokes the gcov tool. -This option is currently only available with the gnu compiler and on a few systems. -To use, submit a full test suite using a version of Icepack on the Consortium master -and the gnu compiler with the ``--codecov`` argument to **icepack.setup**. -The test suite will run and then a report will be generated and uploaded to -the `codecov.io site `_ by the -**report_codecov.csh** script. -This is a special diagnostic test and does not constitute proper model testing. -General use is not recommended, this is mainly used as a diagnostic to periodically -assess test coverage. In addition, the interaction with codecov.io is not always robust. -A sample job submission would look like :: - -$ ./icepack.setup -m conrad -e gnu --suite base_suite,travis_suite,quick_suite --testid cc01 --codecov - From 1a169a667083e4dcde5b6a790d2ab29ca09b692b Mon Sep 17 00:00:00 2001 From: apcraig Date: Tue, 14 Apr 2020 22:11:58 +0000 Subject: [PATCH 26/27] update travis testing and icepack.setup error message --- .travis.yml | 4 ++++ icepack.setup | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 85cc22e4b..bcf525cc3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,6 +27,10 @@ install: #-e 'mirror /archive/Model-Data/CICE/ ~/ICEPACK_INPUTDATA; quit'" script: + # verify icepack.setup --case and icepack.setup --test don't error then run test suite + - "./icepack.setup --case trcase --mach travisCI --env gnu --pes 1x1 -s diag1" + - "./icepack.setup --test smoke --testid trtest --mach travisCI --env gnu + --pes 1x1 -s diag1" - "./icepack.setup --suite travis_suite --testid travisCItest --mach travisCI --env gnu && cd testsuite.travisCItest && diff --git a/icepack.setup b/icepack.setup index 3921d26b1..914bb9540 100755 --- a/icepack.setup +++ b/icepack.setup @@ -374,7 +374,9 @@ endif if ( ${tdir} != ${spval} ) then set tsdir = ${tdir} endif -if (-e $tsfile) then +if (-e ${tsfile}) then + ls -al ${tsfile} + echo "${0}: ERROR in tsfile, ${tsfile}" echo "${0}: ERROR in tsfile, this should never happen" exit -1 endif From dbe8663bbf475d66df3367be6f276e14aaa913fa Mon Sep 17 00:00:00 2001 From: apcraig Date: Tue, 14 Apr 2020 22:23:23 +0000 Subject: [PATCH 27/27] update travis testing to eliminate error by adding sleep between icepack.setup calls --- .travis.yml | 4 ++-- icepack.setup | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index bcf525cc3..bcbcd0382 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,9 +28,9 @@ install: script: # verify icepack.setup --case and icepack.setup --test don't error then run test suite - - "./icepack.setup --case trcase --mach travisCI --env gnu --pes 1x1 -s diag1" + - "./icepack.setup --case trcase --mach travisCI --env gnu --pes 1x1 -s diag1 && sleep 4" - "./icepack.setup --test smoke --testid trtest --mach travisCI --env gnu - --pes 1x1 -s diag1" + --pes 1x1 -s diag1 && sleep 4" - "./icepack.setup --suite travis_suite --testid travisCItest --mach travisCI --env gnu && cd testsuite.travisCItest && diff --git a/icepack.setup b/icepack.setup index 914bb9540..19723c196 100755 --- a/icepack.setup +++ b/icepack.setup @@ -375,8 +375,6 @@ if ( ${tdir} != ${spval} ) then set tsdir = ${tdir} endif if (-e ${tsfile}) then - ls -al ${tsfile} - echo "${0}: ERROR in tsfile, ${tsfile}" echo "${0}: ERROR in tsfile, this should never happen" exit -1 endif