Add Code Coverage Capability (#313)

* add .codecov.yml

* update icepack.setup

* update codecov range

* update codecov.yml

* update codecov.yml

* update icepack.setup

* rename .codecov.yml codecov.yml

* update icepack.setup

* delete .codecov.yml

* update icepack.setup

* update codecov.yml

* update icepack.setup

* update icepack.setup

* update icepack.setup

* update icepack.setup

* update icepack.setup and switch back to .codecov.yml

* update icepack.setup

* move back to codecov.yml

* update icepack.setup

* update code coverage implementation

* code coverage implementation

* update icepack.setup

* update icepack token

* update documentation

* Update documenation

* update travis testing and icepack.setup error message

* update travis testing to eliminate error by adding sleep between icepack.setup calls

.travis.yml

@@ -26,8 +26,11 @@ install:
   #- "lftp ftp://anonymous:[email protected]@ftp.cgd.ucar.edu 
   #-e 'mirror /archive/Model-Data/CICE/ ~/ICEPACK_INPUTDATA; quit'"
 
-
 script:
+  # verify icepack.setup --case and icepack.setup --test don't error then run test suite
+  - "./icepack.setup --case trcase --mach travisCI --env gnu --pes 1x1 -s diag1 && sleep 4"
+  - "./icepack.setup --test smoke --testid trtest --mach travisCI --env gnu 
+    --pes 1x1 -s diag1 && sleep 4"
   - "./icepack.setup --suite travis_suite --testid travisCItest 
     --mach travisCI --env gnu &&
     cd testsuite.travisCItest &&

README.md

@@ -1,5 +1,6 @@
 [![Build Status](https://travis-ci.org/CICE-Consortium/Icepack.svg?branch=master)](https://travis-ci.org/CICE-Consortium/Icepack)
 [![Documentation Status](https://readthedocs.org/projects/cice-consortium-icepack/badge/?version=master)](http://cice-consortium-icepack.readthedocs.io/en/master/?badge=master)
+[![codecov](https://codecov.io/gh/CICE-Consortium/Icepack/branch/master/graph/badge.svg)](https://codecov.io/gh/CICE-Consortium/Icepack)
 
 ## The Icepack sea-ice column model
 This repository contains files for Icepack, the column physics of the sea ice model [CICE][cice]. Icepack is maintained by the CICE Consortium.  For testing purposes and guidance for including Icepack in other sea ice host models, this repository also includes a driver and basic test suite.   

codecov.yml

@@ -0,0 +1,6 @@
+coverage:
+  range: "20...100"
+  round: down
+  precision: 2
+
+comment: false

configuration/scripts/icepack.settings

@@ -77,5 +77,6 @@ setenv TRFED     0         # number of dissolved iron tracers
 
 ### Specialty code
 setenv ICE_BLDDEBUG  false  # build debug flags
+setenv ICE_CODECOV  false   # code coverage flag
 
 

configuration/scripts/machines/Macros.conrad_gnu

@@ -4,17 +4,29 @@
 
 CPP        := ftn -E
 CPPDEFS    := -DFORTRANUNDERSCORE ${ICE_CPPDEFS}
-CFLAGS     := -c -O2 -ffloat-store -march=native
+CFLAGS     := -c -ffloat-store -march=native
 
 FIXEDFLAGS := -ffixed-line-length-132
 FREEFLAGS  := -ffree-form
 FFLAGS     := -ffloat-store -fconvert=swap -fbacktrace -march=native -ffree-line-length-none
 FFLAGS_NOOPT:= -O0
 
 ifeq ($(ICE_BLDDEBUG), true)
-  FFLAGS     += -O0 -g -Wuninitialized -fbounds-check -ffpe-trap=invalid,zero,overflow,underflow
-else
-  FFLAGS     += -O2
+   FFLAGS  += -O0 -g -Wuninitialized -fbounds-check -ffpe-trap=invalid,zero,overflow,underflow
+   CFLAGS  += -O0
+endif
+
+ifeq ($(ICE_CODECOV), true) 
+   FFLAGS  += -O0 -g -fprofile-arcs -ftest-coverage
+   CFLAGS  += -O0 -g -coverage
+   LDFLAGS += -g -ftest-coverage -fprofile-arcs
+endif
+
+ifneq ($(ICE_BLDDEBUG), true) 
+ifneq ($(ICE_CODECOV), true) 
+   FFLAGS  += -O2
+   CFLAGS  += -O2
+endif
 endif
 
 SCC := cc

configuration/scripts/machines/Macros.gaffney_gnu

@@ -4,17 +4,29 @@
 
 CPP        := ftn -E
 CPPDEFS    := -DFORTRANUNDERSCORE ${ICE_CPPDEFS}
-CFLAGS     := -c -O2
+CFLAGS     := -c
 
 FIXEDFLAGS := -ffixed-line-length-132
 FREEFLAGS  := -ffree-form
 FFLAGS     := -fconvert=big-endian -fbacktrace -ffree-line-length-none
 FFLAGS_NOOPT:= -O0
 
 ifeq ($(ICE_BLDDEBUG), true)
-  FFLAGS     += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow
-else
-  FFLAGS     += -O2
+   FFLAGS  += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow
+   CFLAGS  += -O0
+endif
+
+ifeq ($(ICE_CODECOV), true) 
+   FFLAGS  += -O0 -g -fprofile-arcs -ftest-coverage
+   CFLAGS  += -O0 -g -coverage
+   LDFLAGS += -g -ftest-coverage -fprofile-arcs
+endif
+
+ifneq ($(ICE_BLDDEBUG), true) 
+ifneq ($(ICE_CODECOV), true) 
+   FFLAGS  += -O2
+   CFLAGS  += -O2
+endif
 endif
 
 SCC := gcc

configuration/scripts/machines/Macros.gordon_gnu

@@ -4,17 +4,29 @@
 
 CPP        := ftn -E
 CPPDEFS    := -DFORTRANUNDERSCORE ${ICE_CPPDEFS}
-CFLAGS     := -c -O2 -ffloat-store -march=native
+CFLAGS     := -c -ffloat-store -march=native
 
 FIXEDFLAGS := -ffixed-line-length-132
 FREEFLAGS  := -ffree-form
 FFLAGS     := -ffloat-store -fconvert=swap -fbacktrace -march=native -ffree-line-length-none
 FFLAGS_NOOPT:= -O0
 
 ifeq ($(ICE_BLDDEBUG), true)
-  FFLAGS     += -O0 -g -Wuninitialized -fbounds-check -ffpe-trap=invalid,zero,overflow,underflow
-else
-  FFLAGS     += -O2
+   FFLAGS  += -O0 -g -Wuninitialized -fbounds-check -ffpe-trap=invalid,zero,overflow,underflow
+   CFLAGS  += -O0
+endif
+
+ifeq ($(ICE_CODECOV), true) 
+   FFLAGS  += -O0 -g -fprofile-arcs -ftest-coverage
+   CFLAGS  += -O0 -g -coverage
+   LDFLAGS += -g -ftest-coverage -fprofile-arcs
+endif
+
+ifneq ($(ICE_BLDDEBUG), true) 
+ifneq ($(ICE_CODECOV), true) 
+   FFLAGS  += -O2
+   CFLAGS  += -O2
+endif
 endif
 
 SCC := cc

configuration/scripts/machines/Macros.high_Sierra_gnu

@@ -1,5 +1,5 @@
 #==============================================================================
-# Makefile macros for Travis-CI - GCC and openmpi compilers
+# Makefile macros for high_Sierra - GCC and openmpi compilers
 #==============================================================================
 
 CPP        := cpp

configuration/scripts/machines/Macros.izumi_gnu

@@ -4,17 +4,29 @@
 
 CPP        := ftn -E
 CPPDEFS    := -DFORTRANUNDERSCORE ${ICE_CPPDEFS}
-CFLAGS     := -c -O2
+CFLAGS     := -c
 
 FIXEDFLAGS := -ffixed-line-length-132
 FREEFLAGS  := -ffree-form
 FFLAGS     := -fconvert=big-endian -fbacktrace -ffree-line-length-none
 FFLAGS_NOOPT:= -O0
 
 ifeq ($(ICE_BLDDEBUG), true)
-  FFLAGS     += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow
-else
-  FFLAGS     += -O2
+  FFLAGS   += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow
+  CFLAGS   += -O0
+endif
+
+ifeq ($(ICE_CODECOV), true) 
+   FFLAGS  += -O0 -g -fprofile-arcs -ftest-coverage
+   CFLAGS  += -O0 -g -coverage
+   LDFLAGS += -g -ftest-coverage -fprofile-arcs
+endif
+
+ifneq ($(ICE_BLDDEBUG), true) 
+ifneq ($(ICE_CODECOV), true) 
+   FFLAGS  += -O2
+   CFLAGS  += -O2
+endif
 endif
 
 SCC := gcc

configuration/scripts/machines/Macros.koehr_gnu

@@ -1,20 +1,32 @@
 #==============================================================================
-# Macros file for NAVYDSRC koehr, intel compiler
+# Macros file for NAVYDSRC koehr, gnu compiler
 #==============================================================================
 
 CPP        := ftn -E
 CPPDEFS    := -DFORTRANUNDERSCORE ${ICE_CPPDEFS}
-CFLAGS     := -c -O2
+CFLAGS     := -c
 
 FIXEDFLAGS := -ffixed-line-length-132
 FREEFLAGS  := -ffree-form
 FFLAGS     := -fconvert=big-endian -fbacktrace -ffree-line-length-none
 FFLAGS_NOOPT:= -O0
 
 ifeq ($(ICE_BLDDEBUG), true)
-  FFLAGS     += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow
-else
-  FFLAGS     += -O2
+   FFLAGS  += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow
+   CFLAGS  += -O0
+endif
+
+ifeq ($(ICE_CODECOV), true) 
+   FFLAGS  += -O0 -g -fprofile-arcs -ftest-coverage
+   CFLAGS  += -O0 -g -coverage
+   LDFLAGS += -g -ftest-coverage -fprofile-arcs
+endif
+
+ifneq ($(ICE_BLDDEBUG), true) 
+ifneq ($(ICE_CODECOV), true) 
+   FFLAGS  += -O2
+   CFLAGS  += -O2
+endif
 endif
 
 SCC := gcc

configuration/scripts/machines/Macros.onyx_gnu

@@ -4,17 +4,29 @@
 
 CPP        := ftn -E
 CPPDEFS    := -DFORTRANUNDERSCORE ${ICE_CPPDEFS}
-CFLAGS     := -c -O2 -ffloat-store -march=native
+CFLAGS     := -c -ffloat-store -march=native
 
 FIXEDFLAGS := -ffixed-line-length-132
 FREEFLAGS  := -ffree-form
 FFLAGS     := -ffloat-store -fconvert=swap -fbacktrace -march=native -ffree-line-length-none
 FFLAGS_NOOPT:= -O0
 
 ifeq ($(ICE_BLDDEBUG), true)
-  FFLAGS     += -O0 -g -Wuninitialized -fbounds-check -ffpe-trap=invalid,zero,overflow,underflow
-else
-  FFLAGS     += -O2
+   FFLAGS  += -O0 -g -Wuninitialized -fbounds-check -ffpe-trap=invalid,zero,overflow,underflow
+   CFLAGS  += -O0
+endif
+
+ifeq ($(ICE_CODECOV), true) 
+   FFLAGS  += -O0 -g -fprofile-arcs -ftest-coverage
+   CFLAGS  += -O0 -g -coverage
+   LDFLAGS += -g -ftest-coverage -fprofile-arcs
+endif
+
+ifneq ($(ICE_BLDDEBUG), true) 
+ifneq ($(ICE_CODECOV), true) 
+   FFLAGS  += -O2
+   CFLAGS  += -O2
+endif
 endif
 
 SCC := cc

configuration/scripts/machines/Macros.travisCI_gnu

@@ -4,16 +4,28 @@
 
 CPP        := /usr/bin/cpp
 CPPDEFS    := -DFORTRANUNDERSCORE ${ICE_CPPDEFS}
-CFLAGS     := -c -O2 -fp-model precise   -xHost
+CFLAGS     := -c -fp-model precise   -xHost
 
 FIXEDFLAGS := -132
-FFLAGS     :=  -O2 -ffree-line-length-none -fconvert=big-endian -finit-real=nan
+FFLAGS     :=  -ffree-line-length-none -fconvert=big-endian -finit-real=nan
 FFLAGS_NOOPT:= -O0
 
 ifeq ($(ICE_BLDDEBUG), true)
-    FFLAGS     += -O0 -g -Wextra -fbacktrace -fbounds-check -ffpe-trap=zero,overflow
-else
-    FFLAGS     += -O2
+   FFLAGS  += -O0 -g -Wextra -fbacktrace -fbounds-check -ffpe-trap=zero,overflow
+   CFLAGS  += -O0
+endif
+
+ifeq ($(ICE_CODECOV), true) 
+   FFLAGS  += -O0 -g -fprofile-arcs -ftest-coverage
+   CFLAGS  += -O0 -g -coverage
+   LDFLAGS += -g -ftest-coverage -fprofile-arcs
+endif
+
+ifneq ($(ICE_BLDDEBUG), true) 
+ifneq ($(ICE_CODECOV), true) 
+   FFLAGS  += -O2
+   CFLAGS  += -O2
+endif
 endif
 
 SCC := gcc
@@ -29,7 +41,7 @@ ifeq ($(ICE_IOTYPE), netcdf)
     INCLDIR := $(INCLDIR) -I$(NETCDF_PATH)/include
     LIB_NETCDF := $(NETCDF_PATH)/lib
     LIB_PNETCDF := 
-    SLIBS   := -L$(LIB_NETCDF) -lnetcdf 
+    SLIBS   := -L$(LIB_NETCDF) -lnetcdf
 else
     SLIBS   := 
 endif

configuration/scripts/tests/report_results.csh

@@ -39,6 +39,7 @@ set hash = `grep "#hash = " results.log | cut -c 9-`
 set shhash   = `grep "#hshs = " results.log | cut -c 9-`
 set hashuser = `grep "#hshu = " results.log | cut -c 9-`
 set hashdate = `grep "#hshd = " results.log | cut -c 9-`
+set testsuites = `grep "#suit = " results.log | cut -c 9-`
 set cdat = `grep "#date = " results.log | cut -c 9-`
 set ctim = `grep "#time = " results.log | cut -c 9-`
 set user = `grep "#user = " results.log | cut -c 9-`
@@ -56,6 +57,7 @@ set compilers = `grep -v "#" results.log | grep ${mach}_ | cut -d "_" -f 2 | sor
 #echo "debug ${shhash}"
 #echo "debug ${hashuser}"
 #echo "debug ${hashdate}"
+#echo "debug ${testsuites}"
 #echo "debug ${cdat}"
 #echo "debug ${ctim}"
 #echo "debug ${user}"

doc/source/developer_guide/dg_scripts.rst

@@ -91,14 +91,28 @@ Test scripts
 -------------
 
 Under **configuration/scripts/tests** are several files including the scripts to 
-setup the smoke and restart tests (**test_smoke.script**, **test_restart.script*).
-A baseline test script (**baseline.script**) is also there to setup the regression
+setup the various tests, such as smoke and restart tests (**test_smoke.script**, 
+**test_restart.script**).
+and the files that describe which options files are needed for each test 
+(ie. **test_smoke.files**, **test_restart.files**).
+A baseline test script (**baseline.script**) is also there to setup the general regression
 and comparison testing.  That directory also contains the preset test suites 
 (ie. **base_suite.ts**) and a file that supports post-processing on the model
-output (**timeseries.csh**).  
+output (**timeseries.csh**).  There is also a script **report_results.csh** that 
+pushes results from test suites back to the CICE-Consortium test results wiki page.
+
+To add a new test (for example newtest), several files may be needed,
+
+- **configuration/scripts/tests/test_newtest.script** defines how to run the test.  This chunk
+  of script will be incorporated into the case test script
+- **configuration/scripts/tests/test_newtest.files** list the set of options files found in
+  **configuration/scripts/options/** needed to
+  run this test.  Those files will be copied into the test directory when the test is invoked
+  so they are available for the **test_newtest.script** to use.
+- some new files may be needed in **configuration/scripts/options/**.  These could be 
+  relatively generic **set_nml** or **set_env** files, or they could be test specific files 
+  typically carrying a prefix of **test_nml**.
+
+Generating a new test, particularly the **test_newtest.script** usually takes some iteration 
+before it's working properly.
 
-To add a new test, a file associated with that test will need to be added to the
-**configuration/scripts/tests** directory similar to **test_smoke.script** 
-and **test_restart.script**.  In addition, some new options files in 
-**configuration/scripts/options** may need to be added similar to **test_nml.restart1**,
-**test_nml.restart2**, and **set_nml.restart**.  

doc/source/user_guide/ug_testing.rst

@@ -304,6 +304,9 @@ following options are valid for suites,
 ``--report``
   This is only used by ``--suite`` and when set, invokes a script that sends the test results to the results page when all tests are complete.  Please see :ref:`testreporting` for more information.
 
+``--codecov``
+  When invoked, code coverage diagnostics are generated.  This will modify the build and reduce optimization.  The results will be uploaded to the **codecov.io** website via the **report_codecov.csh** script.  General use is not recommended, this is mainly used as a diagnostic to periodically assess test coverage.  Please see :ref:`codecoverage` for more information.
+
 Please see :ref:`case_options` and :ref:`indtests` for more details about how these options are used.
 
 
@@ -493,6 +496,39 @@ The reporting can also be automated in a test suite by adding ``--report`` to ``
 With ``--report``, the suite will create all the tests, build and submit them,
 wait for all runs to be complete, and run the results and report_results scripts.
 
+.. _codecoverage:
+
+Code Coverage Testing
+------------------------
+
+The ``--codecov`` feature in **icepack.setup** provides a method to diagnose code coverage.
+This argument turns on special compiler flags including reduced optimization and then
+invokes the gcov tool.
+This option is currently only available with the gnu compiler and on a few systems.
+To use, submit a full test suite using a version of Icepack on the Consortium master
+and the gnu compiler with the ``--codecov`` argument.
+The test suite will run and then a report will be generated and uploaded to 
+the `codecov.io site <https://codecov.io/gh/CICE-Consortium/Icepack>`_ by the 
+**report_codecov.csh** script.  
+
+This is a special diagnostic test and does not constitute proper model testing.
+General use is not recommended, this is mainly used as a diagnostic to periodically 
+assess test coverage.  The interaction with codecov.io is not always robust and
+can be tricky to manage.  Some constraints are that the output generated at runtime
+is copied into the directory where compilation took place.  That means each
+test should be compiled separately.  Tests that invoke multiple runs
+(such as exact restart) will only save coverage information
+for the last run, so some coverage information may be lost.  The gcov tool can
+be a little slow to run on large test suites, and the codecov.io bash uploader
+(that runs gcov and uploads the data to codecov.io) is constantly evolving.
+Finally, gcov requires that the diagnostic output be copied into the git sandbox for
+analysis.  These constraints are handled by the current scripts, but may change
+in the future.
+
+A sample job submission would look like ::
+
+$ ./icepack.setup -m conrad -e gnu --suite base_suite,travis_suite,quick_suite --testid cc01 --codecov
+
 .. _testplotting:
 
 Test Plotting
@@ -546,3 +582,4 @@ This plotting script can be used to plot the following variables:
   - congelation (m)
   - snow-ice (m)
   - initial energy change (:math:`W/m^2`)
+