Merge branch 'develop' into feature/ocnanalecen

.gitignore

@@ -36,7 +36,6 @@ fix/gsi
 fix/lut
 fix/mom6
 fix/orog
-fix/reg2grb2
 fix/sfc_climo
 fix/ugwd
 fix/verif
@@ -99,6 +98,9 @@ parm/post/postxconfig-NT-GFS-WAFS.txt
 parm/post/postxconfig-NT-GFS.txt
 parm/post/postxconfig-NT-gefs-aerosol.txt
 parm/post/postxconfig-NT-gefs-chem.txt
+parm/post/ocean.csv
+parm/post/ice.csv
+parm/post/ocnicepost.nml.jinja2
 parm/ufs/noahmptable.tbl
 parm/ufs/model_configure.IN
 parm/ufs/MOM_input_*.IN
@@ -137,7 +139,6 @@ sorc/radmon_bcor.fd
 sorc/radmon_time.fd
 sorc/rdbfmsua.fd
 sorc/recentersigp.fd
-sorc/reg2grb2.fd
 sorc/supvit.fd
 sorc/syndat_getjtbul.fd
 sorc/syndat_maksynrc.fd
@@ -147,6 +148,7 @@ sorc/tocsbufr.fd
 sorc/upp.fd
 sorc/vint.fd
 sorc/webtitle.fd
+sorc/ocnicepost.fd
 
 # Ignore scripts from externals
 #------------------------------

Jenkinsfile

@@ -1,15 +1,15 @@
 def Machine = 'none'
 def machine = 'none'
 def HOME = 'none'
-def localworkspace = 'none'
-def commonworkspace = 'none'
+def caseList = ''
+def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/stmp/CI/HERCULES']
 
 pipeline {
     agent { label 'built-in' }
 
     options {
         skipDefaultCheckout()
-        buildDiscarder(logRotator(numToKeepStr: '2'))
+        parallelsAlwaysFailFast()
     }
 
     stages { // This initial stage is used to get the Machine name from the GitHub labels on the PR
@@ -20,7 +20,6 @@ pipeline {
             agent { label 'built-in' }
             steps {
                 script {
-                    localworkspace = env.WORKSPACE
                     machine = 'none'
                     for (label in pullRequest.labels) {
                         echo "Label: ${label}"
@@ -42,17 +41,19 @@ pipeline {
             agent { label "${machine}-emc" }
             steps {
                 script {
-                    properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
-                    HOME = "${WORKSPACE}/TESTDIR"
-                    commonworkspace = "${WORKSPACE}"
-                    sh(script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true)
-                    pullRequest.addLabel("CI-${Machine}-Building")
-                    if (pullRequest.labels.any { value -> value.matches("CI-${Machine}-Ready") }) {
-                        pullRequest.removeLabel("CI-${Machine}-Ready")
+                    ws("${custom_workspace[machine]}/${env.CHANGE_ID}") {
+                        properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
+                        HOME = "${WORKSPACE}"
+                        sh(script: "mkdir -p ${HOME}/RUNTESTS;rm -Rf ${HOME}/RUNTESTS/error.logs")
+                        pullRequest.addLabel("CI-${Machine}-Building")
+                        if (pullRequest.labels.any { value -> value.matches("CI-${Machine}-Ready") }) {
+                            pullRequest.removeLabel("CI-${Machine}-Ready")
+                        }
+                    }
+                    pullRequest.comment("Building and running on ${Machine} in directory ${HOME}")
                 }
             }
         }
-    }
 
         stage('Build System') {
             matrix {
@@ -71,35 +72,42 @@ pipeline {
                         steps {
                             script {
                                 def HOMEgfs = "${HOME}/${system}" // local HOMEgfs is used to build the system on per system basis under the common workspace HOME
-                                sh(script: "mkdir -p ${HOMEgfs}", returnStatus: true)
+                                sh(script: "mkdir -p ${HOMEgfs}")
                                 ws(HOMEgfs) {
-                                    env.MACHINE_ID = machine // MACHINE_ID is used in the build scripts to determine the machine and is added to the shell environment
                                     if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { // if the system is already built, skip the build in the case of re-runs
                                         sh(script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim() // TODO: and user configurable control to manage build semphore
-                                        ws(commonworkspace) { pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") }
+                                        pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}<br>Still doing a checkout to get the latest changes")
+                                        checkout scm
+                                        dir('sorc') {
+                                            sh(script: './link_workflow.sh')
+                                        }
                                     } else {
                                         checkout scm
-                                        sh(script: 'source workflow/gw_setup.sh;which git;git --version;git submodule update --init --recursive', returnStatus: true)
                                         def builds_file = readYaml file: 'ci/cases/yamls/build.yaml'
                                         def build_args_list = builds_file['builds']
                                         def build_args = build_args_list[system].join(' ').trim().replaceAll('null', '')
                                         dir("${HOMEgfs}/sorc") {
-                                            sh(script: "${build_args}", returnStatus: true)
-                                            sh(script: './link_workflow.sh', returnStatus: true)
-                                            sh(script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
+                                            sh(script: "${build_args}")
+                                            sh(script: './link_workflow.sh')
+                                            sh(script: "echo ${HOMEgfs} > BUILT_semaphor")
                                         }
                                     }
-                                    if (pullRequest.labels.any { value -> value.matches("CI-${Machine}-Building") }) {
-                                        pullRequest.removeLabel("CI-${Machine}-Building")
-                                }
-                                    pullRequest.addLabel("CI-${Machine}-Running")
-                            }
+                                    if (env.CHANGE_ID && system == 'gfs') {
+                                       if (pullRequest.labels.any { value -> value.matches("CI-${Machine}-Building") }) {
+                                           pullRequest.removeLabel("CI-${Machine}-Building")
+                                       }
+                                       pullRequest.addLabel("CI-${Machine}-Running")
+                                    }
+                                    if (system == 'gfs') {
+                                        caseList = sh(script: "${HOMEgfs}/ci/scripts/utils/get_host_case_list.py ${machine}", returnStdout: true).trim().split()
+                                    }
+                               }
+                           }
                         }
                     }
                 }
             }
         }
-}
 
         stage('Run Tests') {
             matrix {
@@ -108,40 +116,60 @@ pipeline {
                     axis {
                         name 'Case'
                         // TODO add dynamic list of cases from env vars (needs addtional plugins)
-                        values 'C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW', 'C96_atm3DVar', 'C48mx500_3DVarAOWCDA', 'C96C48_hybatmDA', 'C96_atmsnowDA'
+                        values 'C48C48_ufs_hybatmDA', 'C48_ATM', 'C48_S2SW', 'C48_S2SWA_gefs', 'C48mx500_3DVarAOWCDA', 'C96C48_hybatmDA', 'C96_atm3DVar', 'C96_atmsnowDA'
                     }
                 }
                 stages {
-                    stage('Create Experiment') {
+
+                    stage('Create Experiments') {
+                        when {
+                            expression { return caseList.contains(Case) }
+                        }
                         steps {
                                 script {
-                                    sh(script: "sed -n '/{.*}/!p' ${HOME}/gfs/ci/cases/pr/${Case}.yaml > ${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp", returnStatus: true)
+                                    sh(script: "sed -n '/{.*}/!p' ${HOME}/gfs/ci/cases/pr/${Case}.yaml > ${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp")
                                     def yaml_case = readYaml file: "${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp"
                                     system = yaml_case.experiment.system
                                     def HOMEgfs = "${HOME}/${system}"   // local HOMEgfs is used to populate the XML on per system basis
                                     env.RUNTESTS = "${HOME}/RUNTESTS"
-                                    sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true)
+                                    sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml")
                                 }
                         }
                     }
+
                     stage('Run Experiments') {
+                        when {
+                            expression { return caseList.contains(Case) }
+                        }
                         steps {
                             script {
                                 HOMEgfs = "${HOME}/gfs"  // common HOMEgfs is used to launch the scripts that run the experiments
                                 ws(HOMEgfs) {
                                     pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true).trim()
                                     pullRequest.comment("**Running** experiment: ${Case} on ${Machine}<br>With the experiment in directory:<br>`${HOME}/RUNTESTS/${pslot}`")
                                     try {
-                                        sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: true)
+                                       sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}")
                                     } catch (Exception e) {
                                         pullRequest.comment("**FAILURE** running experiment: ${Case} on ${Machine}")
+                                        sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_all_batch_jobs ${HOME}/RUNTESTS")
+                                        ws(HOME) {
+                                            if (fileExists('RUNTESTS/error.logs')) {
+                                                def fileContent = readFile 'RUNTESTS/error.logs'
+                                                def lines = fileContent.readLines()
+                                                for (line in lines) {
+                                                    echo "archiving: ${line}"
+                                                    archiveArtifacts artifacts: "${line}", fingerprint: true
+                                                }   
+                                            }
+                                        }
                                         error("Failed to run experiments ${Case} on ${Machine}")
                                     }
-                                    pullRequest.comment("**SUCCESS** running experiment: ${Case} on ${Machine}")
                                 }
+                                pullRequest.comment("**SUCCESS** running experiment: ${Case} on ${Machine}")
                             }
                         }
                     }
+
                 }
             }
         }
@@ -175,14 +203,6 @@ pipeline {
                     def timestamp = new Date().format('MM dd HH:mm:ss', TimeZone.getTimeZone('America/New_York'))
                     pullRequest.comment("**CI FAILED** ${Machine} at ${timestamp}<br>Built and ran in directory `${HOME}`")
                 }
-                if (fileExists('${HOME}/RUNTESTS/ci.log')) {
-                    def fileContent = readFile '${HOME}/RUNTESTS/ci.log'
-                    fileContent.eachLine { line ->
-                        if (line.contains('.log')) {
-                            archiveArtifacts artifacts: "${line}", fingerprint: true
-                        }
-                    }
-                }
             }
         }
     }

ci/cases/pr/C48C48_ufs_hybatmDA.yaml

@@ -0,0 +1,22 @@
+experiment:
+  system: gfs
+  mode: cycled
+
+arguments:
+  pslot: {{ 'pslot' | getenv }}
+  app: ATM
+  resdetatmos: 48
+  resensatmos: 48
+  comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
+  expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
+  idate: 2021032312
+  edate: 2021032400
+  nens: 2
+  gfs_cyc: 1
+  start: warm
+  yaml: {{ HOMEgfs }}/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml
+
+skip_ci_on_hosts:
+  - hera
+  - orion
+  - hercules

ci/cases/pr/C48mx500_3DVarAOWCDA.yaml

@@ -19,4 +19,5 @@ arguments:
 
 skip_ci_on_hosts:
   - orion
+  - hera
   - hercules

ci/cases/yamls/atmsnowDA_defaults_ci.yaml

@@ -2,5 +2,5 @@ defaults:
   !INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml
 base:
   DOIAU: "NO"
-  DO_JEDILANDDA: "YES"
+  DO_JEDISNOWDA: "YES"
   ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }}

ci/cases/yamls/build.yaml

@@ -1,3 +1,3 @@
 builds:
  - gefs: './build_all.sh'
- - gfs: './build_all.sh -gu'
+ - gfs: './build_all.sh -wgu'

ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml

@@ -0,0 +1,20 @@
+defaults:
+  !INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml
+base:
+  DOIAU: "NO"
+  DO_JEDIATMVAR: "YES"
+  DO_JEDIATMENS: "YES"
+  ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }}
+atmanl:
+  LAYOUT_X_ATMANL: 1
+  LAYOUT_Y_ATMANL: 1
+atmensanl:
+  LAYOUT_X_ATMENSANL: 1
+  LAYOUT_Y_ATMENSANL: 1
+esfc:
+  DONST: "NO"
+nsst:
+  NST_MODEL: "1"
+sfcanl:
+  DONST: "NO"
+

ci/scripts/check_ci.sh

@@ -8,7 +8,7 @@ set -eux
 #                     to run from within a cron job in the CI Managers account
 #####################################################################################
 
-ROOT_DIR="$(cd "$(dirname  "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd )"
+HOMEgfs="$(cd "$(dirname  "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd )"
 scriptname=$(basename "${BASH_SOURCE[0]}")
 echo "Begin ${scriptname} at $(date -u)" || true
 export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]'
@@ -20,21 +20,22 @@ REPO_URL="https://github.com/NOAA-EMC/global-workflow.git"
 #  Set up runtime environment varibles for accounts on supproted machines
 #########################################################################
 
-source "${ROOT_DIR}/ush/detect_machine.sh"
+source "${HOMEgfs}/ush/detect_machine.sh"
 case ${MACHINE_ID} in
   hera | orion | hercules)
    echo "Running Automated Testing on ${MACHINE_ID}"
-   source "${ROOT_DIR}/ci/platforms/config.${MACHINE_ID}"
+   source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}"
    ;;
  *)
    echo "Unsupported platform. Exiting with error."
    exit 1
    ;;
 esac
 set +x
-source "${ROOT_DIR}/ush/module-setup.sh"
-source "${ROOT_DIR}/ci/scripts/utils/ci_utils.sh"
-module use "${ROOT_DIR}/modulefiles"
+export HOMEgfs
+source "${HOMEgfs}/ush/module-setup.sh"
+source "${HOMEgfs}/ci/scripts/utils/ci_utils.sh"
+module use "${HOMEgfs}/modulefiles"
 module load "module_gwsetup.${MACHINE_ID}"
 module list
 set -x
@@ -57,7 +58,7 @@ pr_list_dbfile="${GFS_CI_ROOT}/open_pr_list.db"
 
 pr_list=""
 if [[ -f "${pr_list_dbfile}" ]]; then
-  pr_list=$("${ROOT_DIR}/ci/scripts/pr_list_database.py" --dbfile "${pr_list_dbfile}" --display | grep -v Failed | grep Running | awk '{print $1}') || true
+  pr_list=$("${HOMEgfs}/ci/scripts/pr_list_database.py" --dbfile "${pr_list_dbfile}" --display | grep -v Failed | grep Running | awk '{print $1}') || true
 fi
 if [[ -z "${pr_list+x}" ]]; then
   echo "no PRs open and ready to run cases on .. exiting"
@@ -89,13 +90,13 @@ for pr in ${pr_list}; do
     sed -i "1 i\`\`\`" "${output_ci}"
     sed -i "1 i\All CI Test Cases Passed on ${MACHINE_ID^}:" "${output_ci}"
     "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}"
-    "${ROOT_DIR}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
+    "${HOMEgfs}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
     # Check to see if this PR that was opened by the weekly tests and if so close it if it passed on all platforms
     weekly_labels=$(${GH} pr view "${pr}" --repo "${REPO_URL}"  --json headRefName,labels,author --jq 'select(.author.login | contains("emcbot")) | select(.headRefName | contains("weekly_ci")) | .labels[].name ') || true
     if [[ -n "${weekly_labels}" ]]; then
-      num_platforms=$(find "${ROOT_DIR}/ci/platforms" -type f -name "config.*" | wc -l)
+      num_platforms=$(find "${HOMEgfs}/ci/platforms" -type f -name "config.*" | wc -l)
       passed=0
-      for platforms in "${ROOT_DIR}"/ci/platforms/config.*; do
+      for platforms in "${HOMEgfs}"/ci/platforms/config.*; do
         machine=$(basename "${platforms}" | cut -d. -f2)
         if [[ "${weekly_labels}" == *"CI-${machine^}-Passed"* ]]; then
           ((passed=passed+1))
@@ -139,7 +140,7 @@ for pr in ${pr_list}; do
       } >> "${output_ci}"
       sed -i "1 i\`\`\`" "${output_ci}"
       "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}"
-      "${ROOT_DIR}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
+      "${HOMEgfs}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
       for kill_cases in "${pr_dir}/RUNTESTS/"*; do
          pslot=$(basename "${kill_cases}")
          cancel_slurm_jobs "${pslot}"

ci/scripts/clone-build_ci.sh

@@ -74,7 +74,7 @@ set +e
 source "${HOMEgfs}/ush/module-setup.sh"
 export BUILD_JOBS=8
 rm -rf log.build
-./build_all.sh -gu  >> log.build 2>&1
+./build_all.sh -guw  >> log.build 2>&1
 build_status=$?
 
 DATE=$(date +'%D %r')

ci/scripts/driver.sh

@@ -47,12 +47,15 @@ esac
 ######################################################
 # setup runtime env for correct python install and git
 ######################################################
+HOMEgfs=${ROOT_DIR}
+export HOMEgfs
 set +x
 source "${ROOT_DIR}/ci/scripts/utils/ci_utils.sh"
 source "${ROOT_DIR}/ush/module-setup.sh"
 module use "${ROOT_DIR}/modulefiles"
 module load "module_gwsetup.${MACHINE_ID}"
 set -x
+unset HOMEgfs
 
 ############################################################
 # query repo and get list of open PRs with tags {machine}-CI