From b25d616c3e8a31fbd28dc1c0a84d796c09f4a010 Mon Sep 17 00:00:00 2001 From: Tony Craig Date: Fri, 17 Apr 2020 10:05:23 -0700 Subject: [PATCH] Add Code Coverage Capability (#313) * add .codecov.yml * update icepack.setup * update codecov range * update codecov.yml * update codecov.yml * update icepack.setup * rename .codecov.yml codecov.yml * update icepack.setup * delete .codecov.yml * update icepack.setup * update codecov.yml * update icepack.setup * update icepack.setup * update icepack.setup * update icepack.setup * update icepack.setup and switch back to .codecov.yml * update icepack.setup * move back to codecov.yml * update icepack.setup * update code coverage implementation * code coverage implementation * update icepack.setup * update icepack token * update documentation * Update documenation * update travis testing and icepack.setup error message * update travis testing to eliminate error by adding sleep between icepack.setup calls --- .travis.yml | 5 +- README.md | 1 + codecov.yml | 6 ++ configuration/scripts/icepack.settings | 1 + .../scripts/machines/Macros.conrad_gnu | 20 +++- .../scripts/machines/Macros.gaffney_gnu | 20 +++- .../scripts/machines/Macros.gordon_gnu | 20 +++- .../scripts/machines/Macros.high_Sierra_gnu | 2 +- .../scripts/machines/Macros.izumi_gnu | 20 +++- .../scripts/machines/Macros.koehr_gnu | 22 ++++- .../scripts/machines/Macros.onyx_gnu | 20 +++- .../scripts/machines/Macros.travisCI_gnu | 24 +++-- .../scripts/tests/report_results.csh | 2 + doc/source/developer_guide/dg_scripts.rst | 30 ++++-- doc/source/user_guide/ug_testing.rst | 37 ++++++++ icepack.setup | 91 +++++++++++++++++-- 16 files changed, 270 insertions(+), 51 deletions(-) create mode 100644 codecov.yml diff --git a/.travis.yml b/.travis.yml index ca4b34b5b..bcbcd0382 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,8 +26,11 @@ install: #- "lftp ftp://anonymous:travis@travis-ci.org@ftp.cgd.ucar.edu #-e 'mirror /archive/Model-Data/CICE/ ~/ICEPACK_INPUTDATA; quit'" - script: + # verify icepack.setup --case and icepack.setup --test don't error then run test suite + - "./icepack.setup --case trcase --mach travisCI --env gnu --pes 1x1 -s diag1 && sleep 4" + - "./icepack.setup --test smoke --testid trtest --mach travisCI --env gnu + --pes 1x1 -s diag1 && sleep 4" - "./icepack.setup --suite travis_suite --testid travisCItest --mach travisCI --env gnu && cd testsuite.travisCItest && diff --git a/README.md b/README.md index 16ba9ac7e..4a37dc3e4 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ [![Build Status](https://travis-ci.org/CICE-Consortium/Icepack.svg?branch=master)](https://travis-ci.org/CICE-Consortium/Icepack) [![Documentation Status](https://readthedocs.org/projects/cice-consortium-icepack/badge/?version=master)](http://cice-consortium-icepack.readthedocs.io/en/master/?badge=master) +[![codecov](https://codecov.io/gh/CICE-Consortium/Icepack/branch/master/graph/badge.svg)](https://codecov.io/gh/CICE-Consortium/Icepack) ## The Icepack sea-ice column model This repository contains files for Icepack, the column physics of the sea ice model [CICE][cice]. Icepack is maintained by the CICE Consortium. For testing purposes and guidance for including Icepack in other sea ice host models, this repository also includes a driver and basic test suite. diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 000000000..e7f6a6e2f --- /dev/null +++ b/codecov.yml @@ -0,0 +1,6 @@ +coverage: + range: "20...100" + round: down + precision: 2 + +comment: false diff --git a/configuration/scripts/icepack.settings b/configuration/scripts/icepack.settings index 8860c103d..295cacfa2 100755 --- a/configuration/scripts/icepack.settings +++ b/configuration/scripts/icepack.settings @@ -77,5 +77,6 @@ setenv TRFED 0 # number of dissolved iron tracers ### Specialty code setenv ICE_BLDDEBUG false # build debug flags +setenv ICE_CODECOV false # code coverage flag diff --git a/configuration/scripts/machines/Macros.conrad_gnu b/configuration/scripts/machines/Macros.conrad_gnu index 2a62f7de5..dba7798f6 100644 --- a/configuration/scripts/machines/Macros.conrad_gnu +++ b/configuration/scripts/machines/Macros.conrad_gnu @@ -4,7 +4,7 @@ CPP := ftn -E CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} -CFLAGS := -c -O2 -ffloat-store -march=native +CFLAGS := -c -ffloat-store -march=native FIXEDFLAGS := -ffixed-line-length-132 FREEFLAGS := -ffree-form @@ -12,9 +12,21 @@ FFLAGS := -ffloat-store -fconvert=swap -fbacktrace -march=native -ffree-line FFLAGS_NOOPT:= -O0 ifeq ($(ICE_BLDDEBUG), true) - FFLAGS += -O0 -g -Wuninitialized -fbounds-check -ffpe-trap=invalid,zero,overflow,underflow -else - FFLAGS += -O2 + FFLAGS += -O0 -g -Wuninitialized -fbounds-check -ffpe-trap=invalid,zero,overflow,underflow + CFLAGS += -O0 +endif + +ifeq ($(ICE_CODECOV), true) + FFLAGS += -O0 -g -fprofile-arcs -ftest-coverage + CFLAGS += -O0 -g -coverage + LDFLAGS += -g -ftest-coverage -fprofile-arcs +endif + +ifneq ($(ICE_BLDDEBUG), true) +ifneq ($(ICE_CODECOV), true) + FFLAGS += -O2 + CFLAGS += -O2 +endif endif SCC := cc diff --git a/configuration/scripts/machines/Macros.gaffney_gnu b/configuration/scripts/machines/Macros.gaffney_gnu index 0047f951d..febf07bd4 100644 --- a/configuration/scripts/machines/Macros.gaffney_gnu +++ b/configuration/scripts/machines/Macros.gaffney_gnu @@ -4,7 +4,7 @@ CPP := ftn -E CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} -CFLAGS := -c -O2 +CFLAGS := -c FIXEDFLAGS := -ffixed-line-length-132 FREEFLAGS := -ffree-form @@ -12,9 +12,21 @@ FFLAGS := -fconvert=big-endian -fbacktrace -ffree-line-length-none FFLAGS_NOOPT:= -O0 ifeq ($(ICE_BLDDEBUG), true) - FFLAGS += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow -else - FFLAGS += -O2 + FFLAGS += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow + CFLAGS += -O0 +endif + +ifeq ($(ICE_CODECOV), true) + FFLAGS += -O0 -g -fprofile-arcs -ftest-coverage + CFLAGS += -O0 -g -coverage + LDFLAGS += -g -ftest-coverage -fprofile-arcs +endif + +ifneq ($(ICE_BLDDEBUG), true) +ifneq ($(ICE_CODECOV), true) + FFLAGS += -O2 + CFLAGS += -O2 +endif endif SCC := gcc diff --git a/configuration/scripts/machines/Macros.gordon_gnu b/configuration/scripts/machines/Macros.gordon_gnu index 1fcf530da..91b19cbfa 100644 --- a/configuration/scripts/machines/Macros.gordon_gnu +++ b/configuration/scripts/machines/Macros.gordon_gnu @@ -4,7 +4,7 @@ CPP := ftn -E CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} -CFLAGS := -c -O2 -ffloat-store -march=native +CFLAGS := -c -ffloat-store -march=native FIXEDFLAGS := -ffixed-line-length-132 FREEFLAGS := -ffree-form @@ -12,9 +12,21 @@ FFLAGS := -ffloat-store -fconvert=swap -fbacktrace -march=native -ffree-line FFLAGS_NOOPT:= -O0 ifeq ($(ICE_BLDDEBUG), true) - FFLAGS += -O0 -g -Wuninitialized -fbounds-check -ffpe-trap=invalid,zero,overflow,underflow -else - FFLAGS += -O2 + FFLAGS += -O0 -g -Wuninitialized -fbounds-check -ffpe-trap=invalid,zero,overflow,underflow + CFLAGS += -O0 +endif + +ifeq ($(ICE_CODECOV), true) + FFLAGS += -O0 -g -fprofile-arcs -ftest-coverage + CFLAGS += -O0 -g -coverage + LDFLAGS += -g -ftest-coverage -fprofile-arcs +endif + +ifneq ($(ICE_BLDDEBUG), true) +ifneq ($(ICE_CODECOV), true) + FFLAGS += -O2 + CFLAGS += -O2 +endif endif SCC := cc diff --git a/configuration/scripts/machines/Macros.high_Sierra_gnu b/configuration/scripts/machines/Macros.high_Sierra_gnu index df2911f66..d2a0b257e 100644 --- a/configuration/scripts/machines/Macros.high_Sierra_gnu +++ b/configuration/scripts/machines/Macros.high_Sierra_gnu @@ -1,5 +1,5 @@ #============================================================================== -# Makefile macros for Travis-CI - GCC and openmpi compilers +# Makefile macros for high_Sierra - GCC and openmpi compilers #============================================================================== CPP := cpp diff --git a/configuration/scripts/machines/Macros.izumi_gnu b/configuration/scripts/machines/Macros.izumi_gnu index d42cce39e..48669c1b6 100644 --- a/configuration/scripts/machines/Macros.izumi_gnu +++ b/configuration/scripts/machines/Macros.izumi_gnu @@ -4,7 +4,7 @@ CPP := ftn -E CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} -CFLAGS := -c -O2 +CFLAGS := -c FIXEDFLAGS := -ffixed-line-length-132 FREEFLAGS := -ffree-form @@ -12,9 +12,21 @@ FFLAGS := -fconvert=big-endian -fbacktrace -ffree-line-length-none FFLAGS_NOOPT:= -O0 ifeq ($(ICE_BLDDEBUG), true) - FFLAGS += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow -else - FFLAGS += -O2 + FFLAGS += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow + CFLAGS += -O0 +endif + +ifeq ($(ICE_CODECOV), true) + FFLAGS += -O0 -g -fprofile-arcs -ftest-coverage + CFLAGS += -O0 -g -coverage + LDFLAGS += -g -ftest-coverage -fprofile-arcs +endif + +ifneq ($(ICE_BLDDEBUG), true) +ifneq ($(ICE_CODECOV), true) + FFLAGS += -O2 + CFLAGS += -O2 +endif endif SCC := gcc diff --git a/configuration/scripts/machines/Macros.koehr_gnu b/configuration/scripts/machines/Macros.koehr_gnu index 97950c249..b34dfa3ec 100644 --- a/configuration/scripts/machines/Macros.koehr_gnu +++ b/configuration/scripts/machines/Macros.koehr_gnu @@ -1,10 +1,10 @@ #============================================================================== -# Macros file for NAVYDSRC koehr, intel compiler +# Macros file for NAVYDSRC koehr, gnu compiler #============================================================================== CPP := ftn -E CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} -CFLAGS := -c -O2 +CFLAGS := -c FIXEDFLAGS := -ffixed-line-length-132 FREEFLAGS := -ffree-form @@ -12,9 +12,21 @@ FFLAGS := -fconvert=big-endian -fbacktrace -ffree-line-length-none FFLAGS_NOOPT:= -O0 ifeq ($(ICE_BLDDEBUG), true) - FFLAGS += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow -else - FFLAGS += -O2 + FFLAGS += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow + CFLAGS += -O0 +endif + +ifeq ($(ICE_CODECOV), true) + FFLAGS += -O0 -g -fprofile-arcs -ftest-coverage + CFLAGS += -O0 -g -coverage + LDFLAGS += -g -ftest-coverage -fprofile-arcs +endif + +ifneq ($(ICE_BLDDEBUG), true) +ifneq ($(ICE_CODECOV), true) + FFLAGS += -O2 + CFLAGS += -O2 +endif endif SCC := gcc diff --git a/configuration/scripts/machines/Macros.onyx_gnu b/configuration/scripts/machines/Macros.onyx_gnu index d22562760..ba18e077e 100644 --- a/configuration/scripts/machines/Macros.onyx_gnu +++ b/configuration/scripts/machines/Macros.onyx_gnu @@ -4,7 +4,7 @@ CPP := ftn -E CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} -CFLAGS := -c -O2 -ffloat-store -march=native +CFLAGS := -c -ffloat-store -march=native FIXEDFLAGS := -ffixed-line-length-132 FREEFLAGS := -ffree-form @@ -12,9 +12,21 @@ FFLAGS := -ffloat-store -fconvert=swap -fbacktrace -march=native -ffree-line FFLAGS_NOOPT:= -O0 ifeq ($(ICE_BLDDEBUG), true) - FFLAGS += -O0 -g -Wuninitialized -fbounds-check -ffpe-trap=invalid,zero,overflow,underflow -else - FFLAGS += -O2 + FFLAGS += -O0 -g -Wuninitialized -fbounds-check -ffpe-trap=invalid,zero,overflow,underflow + CFLAGS += -O0 +endif + +ifeq ($(ICE_CODECOV), true) + FFLAGS += -O0 -g -fprofile-arcs -ftest-coverage + CFLAGS += -O0 -g -coverage + LDFLAGS += -g -ftest-coverage -fprofile-arcs +endif + +ifneq ($(ICE_BLDDEBUG), true) +ifneq ($(ICE_CODECOV), true) + FFLAGS += -O2 + CFLAGS += -O2 +endif endif SCC := cc diff --git a/configuration/scripts/machines/Macros.travisCI_gnu b/configuration/scripts/machines/Macros.travisCI_gnu index 1ef6b44a2..6e20c2beb 100644 --- a/configuration/scripts/machines/Macros.travisCI_gnu +++ b/configuration/scripts/machines/Macros.travisCI_gnu @@ -4,16 +4,28 @@ CPP := /usr/bin/cpp CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} -CFLAGS := -c -O2 -fp-model precise -xHost +CFLAGS := -c -fp-model precise -xHost FIXEDFLAGS := -132 -FFLAGS := -O2 -ffree-line-length-none -fconvert=big-endian -finit-real=nan +FFLAGS := -ffree-line-length-none -fconvert=big-endian -finit-real=nan FFLAGS_NOOPT:= -O0 ifeq ($(ICE_BLDDEBUG), true) - FFLAGS += -O0 -g -Wextra -fbacktrace -fbounds-check -ffpe-trap=zero,overflow -else - FFLAGS += -O2 + FFLAGS += -O0 -g -Wextra -fbacktrace -fbounds-check -ffpe-trap=zero,overflow + CFLAGS += -O0 +endif + +ifeq ($(ICE_CODECOV), true) + FFLAGS += -O0 -g -fprofile-arcs -ftest-coverage + CFLAGS += -O0 -g -coverage + LDFLAGS += -g -ftest-coverage -fprofile-arcs +endif + +ifneq ($(ICE_BLDDEBUG), true) +ifneq ($(ICE_CODECOV), true) + FFLAGS += -O2 + CFLAGS += -O2 +endif endif SCC := gcc @@ -29,7 +41,7 @@ ifeq ($(ICE_IOTYPE), netcdf) INCLDIR := $(INCLDIR) -I$(NETCDF_PATH)/include LIB_NETCDF := $(NETCDF_PATH)/lib LIB_PNETCDF := - SLIBS := -L$(LIB_NETCDF) -lnetcdf + SLIBS := -L$(LIB_NETCDF) -lnetcdf else SLIBS := endif diff --git a/configuration/scripts/tests/report_results.csh b/configuration/scripts/tests/report_results.csh index a52294a8a..141bc45c2 100755 --- a/configuration/scripts/tests/report_results.csh +++ b/configuration/scripts/tests/report_results.csh @@ -39,6 +39,7 @@ set hash = `grep "#hash = " results.log | cut -c 9-` set shhash = `grep "#hshs = " results.log | cut -c 9-` set hashuser = `grep "#hshu = " results.log | cut -c 9-` set hashdate = `grep "#hshd = " results.log | cut -c 9-` +set testsuites = `grep "#suit = " results.log | cut -c 9-` set cdat = `grep "#date = " results.log | cut -c 9-` set ctim = `grep "#time = " results.log | cut -c 9-` set user = `grep "#user = " results.log | cut -c 9-` @@ -56,6 +57,7 @@ set compilers = `grep -v "#" results.log | grep ${mach}_ | cut -d "_" -f 2 | sor #echo "debug ${shhash}" #echo "debug ${hashuser}" #echo "debug ${hashdate}" +#echo "debug ${testsuites}" #echo "debug ${cdat}" #echo "debug ${ctim}" #echo "debug ${user}" diff --git a/doc/source/developer_guide/dg_scripts.rst b/doc/source/developer_guide/dg_scripts.rst index e76c8cac6..fe6b6d613 100755 --- a/doc/source/developer_guide/dg_scripts.rst +++ b/doc/source/developer_guide/dg_scripts.rst @@ -91,14 +91,28 @@ Test scripts ------------- Under **configuration/scripts/tests** are several files including the scripts to -setup the smoke and restart tests (**test_smoke.script**, **test_restart.script*). -A baseline test script (**baseline.script**) is also there to setup the regression +setup the various tests, such as smoke and restart tests (**test_smoke.script**, +**test_restart.script**). +and the files that describe which options files are needed for each test +(ie. **test_smoke.files**, **test_restart.files**). +A baseline test script (**baseline.script**) is also there to setup the general regression and comparison testing. That directory also contains the preset test suites (ie. **base_suite.ts**) and a file that supports post-processing on the model -output (**timeseries.csh**). +output (**timeseries.csh**). There is also a script **report_results.csh** that +pushes results from test suites back to the CICE-Consortium test results wiki page. + +To add a new test (for example newtest), several files may be needed, + +- **configuration/scripts/tests/test_newtest.script** defines how to run the test. This chunk + of script will be incorporated into the case test script +- **configuration/scripts/tests/test_newtest.files** list the set of options files found in + **configuration/scripts/options/** needed to + run this test. Those files will be copied into the test directory when the test is invoked + so they are available for the **test_newtest.script** to use. +- some new files may be needed in **configuration/scripts/options/**. These could be + relatively generic **set_nml** or **set_env** files, or they could be test specific files + typically carrying a prefix of **test_nml**. + +Generating a new test, particularly the **test_newtest.script** usually takes some iteration +before it's working properly. -To add a new test, a file associated with that test will need to be added to the -**configuration/scripts/tests** directory similar to **test_smoke.script** -and **test_restart.script**. In addition, some new options files in -**configuration/scripts/options** may need to be added similar to **test_nml.restart1**, -**test_nml.restart2**, and **set_nml.restart**. diff --git a/doc/source/user_guide/ug_testing.rst b/doc/source/user_guide/ug_testing.rst index 34a34086c..5deb40042 100755 --- a/doc/source/user_guide/ug_testing.rst +++ b/doc/source/user_guide/ug_testing.rst @@ -304,6 +304,9 @@ following options are valid for suites, ``--report`` This is only used by ``--suite`` and when set, invokes a script that sends the test results to the results page when all tests are complete. Please see :ref:`testreporting` for more information. +``--codecov`` + When invoked, code coverage diagnostics are generated. This will modify the build and reduce optimization. The results will be uploaded to the **codecov.io** website via the **report_codecov.csh** script. General use is not recommended, this is mainly used as a diagnostic to periodically assess test coverage. Please see :ref:`codecoverage` for more information. + Please see :ref:`case_options` and :ref:`indtests` for more details about how these options are used. @@ -493,6 +496,39 @@ The reporting can also be automated in a test suite by adding ``--report`` to `` With ``--report``, the suite will create all the tests, build and submit them, wait for all runs to be complete, and run the results and report_results scripts. +.. _codecoverage: + +Code Coverage Testing +------------------------ + +The ``--codecov`` feature in **icepack.setup** provides a method to diagnose code coverage. +This argument turns on special compiler flags including reduced optimization and then +invokes the gcov tool. +This option is currently only available with the gnu compiler and on a few systems. +To use, submit a full test suite using a version of Icepack on the Consortium master +and the gnu compiler with the ``--codecov`` argument. +The test suite will run and then a report will be generated and uploaded to +the `codecov.io site `_ by the +**report_codecov.csh** script. + +This is a special diagnostic test and does not constitute proper model testing. +General use is not recommended, this is mainly used as a diagnostic to periodically +assess test coverage. The interaction with codecov.io is not always robust and +can be tricky to manage. Some constraints are that the output generated at runtime +is copied into the directory where compilation took place. That means each +test should be compiled separately. Tests that invoke multiple runs +(such as exact restart) will only save coverage information +for the last run, so some coverage information may be lost. The gcov tool can +be a little slow to run on large test suites, and the codecov.io bash uploader +(that runs gcov and uploads the data to codecov.io) is constantly evolving. +Finally, gcov requires that the diagnostic output be copied into the git sandbox for +analysis. These constraints are handled by the current scripts, but may change +in the future. + +A sample job submission would look like :: + +$ ./icepack.setup -m conrad -e gnu --suite base_suite,travis_suite,quick_suite --testid cc01 --codecov + .. _testplotting: Test Plotting @@ -546,3 +582,4 @@ This plotting script can be used to plot the following variables: - congelation (m) - snow-ice (m) - initial energy change (:math:`W/m^2`) + diff --git a/icepack.setup b/icepack.setup index 2f54cfa2b..19723c196 100755 --- a/icepack.setup +++ b/icepack.setup @@ -34,6 +34,8 @@ set stime = `date -u "+%H%M%S"` set docase = 0 set dotest = 0 set dosuite = 0 +set codecov = 0 # code coverage measurement and reporting +set codecovflag = false if ($#argv < 1) then set helpheader = 1 @@ -80,7 +82,7 @@ SYNOPSIS --suite SUITE[,SUITE2] -m MACH --testid ID [-e ENV1,ENV2][--acct ACCT][--bdir DIR][--bgen DIR] - [--bcmp DIR][--tdir PATH][--report] + [--bcmp DIR][--tdir PATH][--report || --codecov] DESCRIPTION --help, -h : help @@ -107,6 +109,8 @@ DESCRIPTION --testid : test ID, user-defined id for testing (REQUIRED with --test or --suite) --diff : generate comparison against another case --report : automatically post results when tests are complete + --codecov : generate and report test coverage metrics when tests are complete, + requires GNU compiler (--env gnu) EXAMPLES icepack.setup -c caseB -m gordon -e cray -s diag1,debug @@ -235,15 +239,20 @@ while (1) set report = 1 shift argv + else if ("$option" == "--codecov") then + set codecov = 1 + set codecovflag = true + shift argv + # arguments with settings else shift argv if ( $#argv < 1 ) then - echo "${0}: ERROR1 in $option" + echo "${0}: ERROR in $option, unsupported or missing an argument" exit -1 endif - if ($argv[1] =~ $dash* ) then - echo "${0}: ERROR2 in $option" + if ("$argv[1]" =~ "$dash*" ) then + echo "${0}: ERROR in $option, possibly missing an argument" exit -1 endif @@ -308,7 +317,30 @@ if (${dosum} > 1) then exit -1 endif +if ($codecov == 1 && $report == 1) then + echo "${0}: ERROR in arguments, not recommmended to set both --codecov and --report" + exit -1 +endif + +if ($codecov == 1 && "$compilers" != "gnu") then + echo "${0}: ERROR in arguments, must use --env gnu with --codecov" + exit -1 +endif + +if ($codecov == 1 && `where curl` == "" && `where wget` == "") then + echo "${0}: ERROR 'curl' or 'wget' is required for --codecov" + exit -1 +endif + if (${dosuite} == 0) then + if ($report == 1) then + echo "${0}: ERROR in arguments, must use --suite with --report" + exit -1 + endif + if ($codecov == 1) then + echo "${0}: ERROR in arguments, must use --suite with --codecov" + exit -1 + endif if ("$compilers" =~ "*,*") then echo "${0}: ERROR in arguments, cannot set multiple compilers without --suite" exit -1 @@ -342,7 +374,7 @@ endif if ( ${tdir} != ${spval} ) then set tsdir = ${tdir} endif -if (-e $tsfile) then +if (-e ${tsfile}) then echo "${0}: ERROR in tsfile, this should never happen" exit -1 endif @@ -388,10 +420,7 @@ else endif cp -f ${ICE_SCRIPTS}/tests/report_results.csh ${tsdir} cp -f ${ICE_SCRIPTS}/tests/timeseries.csh ${tsdir} - - if ($report == 1) then - cp -f ${ICE_SCRIPTS}/tests/poll_queue.csh ${tsdir} - endif + cp -f ${ICE_SCRIPTS}/tests/poll_queue.csh ${tsdir} cat >! ${tsdir}/suite.run << EOF0 #!/bin/csh -f @@ -417,16 +446,29 @@ echo "#hash = ${hash}" >> results.log echo "#hshs = ${shhash}" >> results.log echo "#hshu = ${hashuser}" >> results.log echo "#hshd = ${hashdate}" >> results.log +echo "#suit = ${testsuite}" >> results.log echo "#date = ${cdate}" >> results.log echo "#time = ${ctime}" >> results.log echo "#mach = ${machine}" >> results.log echo "#user = ${user}" >> results.log echo "#vers = ${vers}" >> results.log echo "#------- " >> results.log +EOF0 + +cat >! ${tsdir}/report_codecov.csh << EOF0 +#!/bin/csh -f + +setenv CODECOV_TOKEN "df12b574-8dce-439d-8d3b-ed7428d7598a" # consortium icepack +#setenv CODECOV_TOKEN "0b3feb13-0110-4618-821e-248c2f1f7cf3" # apcraig icepack +set report_name = "${shhash}:${branch}:${machine} ${testsuite}" + +set use_curl = 1 + EOF0 chmod +x ${tsdir}/suite.run chmod +x ${tsdir}/results.csh + chmod +x ${tsdir}/report_codecov.csh endif @@ -581,6 +623,8 @@ foreach compiler ( $ncompilers ) endif endif + set rundir = ${ICE_MACHINE_WKDIR}/${casename} + #------------------------------------------------------------ # Compute a default blocksize @@ -613,6 +657,7 @@ foreach compiler ( $ncompilers ) echo "ICE_CASEDIR = ${casedir}" echo "ICE_MACHINE = ${machine}" echo "ICE_COMPILER = ${compiler}" + echo "ICE_RUNDIR = ${rundir}" #------------------------------------------------------------ # Copy in and update icepack.settings and icepack_in files @@ -658,7 +703,7 @@ setenv ICE_CASEDIR ${casedir} setenv ICE_MACHINE ${machine} setenv ICE_COMPILER ${compiler} setenv ICE_MACHCOMP ${machcomp} -setenv ICE_RUNDIR ${ICE_MACHINE_WKDIR}/${casename} +setenv ICE_RUNDIR ${rundir} setenv ICE_GRID ${grid} setenv ICE_NXGLOB ${ICE_DECOMP_NXGLOB} setenv ICE_NTASKS ${task} @@ -673,6 +718,7 @@ setenv ICE_TESTNAME ${testname_noid} setenv ICE_BFBCOMP ${fbfbcomp} setenv ICE_ACCOUNT ${acct} setenv ICE_QUEUE ${queue} +setenv ICE_CODECOV ${codecovflag} EOF1 if (${sets} != "") then @@ -805,6 +851,12 @@ cat >> ${tsdir}/results.csh << EOF cat ${testname_base}/test_output >> results.log EOF +cat >> ${tsdir}/report_codecov.csh << EOF +mkdir ${testname_base}/codecov_output +cp ${rundir}/compile/*.{gcno,gcda} ${testname_base}/codecov_output/ + +EOF + cat >> ${tsdir}/suite.run << EOF cd ${testname_base} ./icepack.build @@ -883,6 +935,20 @@ echo "\$failures of \$chkcnt tests FAILED" #echo " \$failbfbc of \$failures FAILED bfbcomp" #echo " \$failgen of \$failures FAILED generate" exit \$failures +EOF + +cat >> ${tsdir}/report_codecov.csh << EOF +source ${ICE_SCRIPTS}/machines/env.${machcomp} + +if ( \${use_curl} == 1 ) then + bash -c "bash <(curl -s https://codecov.io/bash) -n '\${report_name}' -y ./codecov.yml " +else + bash -c "bash <(wget -O - https://codecov.io/bash) -n '\${report_name}' -y ./codecov.yml " +endif + +sleep 10 +rm -r -f ./*/codecov_output + EOF # build and submit tests @@ -894,6 +960,11 @@ EOF ./results.csh ./report_results.csh endif + if ($codecov == 1) then + echo "Generating codecov reports" + ./poll_queue.csh + ./report_codecov.csh + endif cd ${ICE_SANDBOX} endif