Merge branch 'NOAA-EMC:develop' into develop

.gitignore

@@ -43,8 +43,12 @@ fix/wave
 
 # Ignore parm file symlinks
 #--------------------------
-parm/config/config.base
-parm/gldas
+parm/gdas/aero
+parm/gdas/atm
+parm/gdas/io
+parm/gdas/ioda
+parm/gdas/snow
+parm/gdas/soca
 parm/monitor
 parm/post/AEROSOL_LUTS.dat
 parm/post/nam_micro_lookup.dat

.gitmodules

@@ -1,28 +1,28 @@
 [submodule "sorc/ufs_model.fd"]
-	path = sorc/ufs_model.fd
-	url = https://github.com/ufs-community/ufs-weather-model
-	ignore = dirty
+  path = sorc/ufs_model.fd
+  url = https://github.com/ufs-community/ufs-weather-model
+  ignore = dirty
 [submodule "sorc/wxflow"]
-	path = sorc/wxflow
-	url = https://github.com/NOAA-EMC/wxflow
+  path = sorc/wxflow
+  url = https://github.com/NOAA-EMC/wxflow
 [submodule "sorc/gfs_utils.fd"]
-	path = sorc/gfs_utils.fd
-	url = https://github.com/NOAA-EMC/gfs-utils
+  path = sorc/gfs_utils.fd
+  url = https://github.com/NOAA-EMC/gfs-utils
 [submodule "sorc/ufs_utils.fd"]
-	path = sorc/ufs_utils.fd
-	url = https://github.com/ufs-community/UFS_UTILS.git
+  path = sorc/ufs_utils.fd
+  url = https://github.com/ufs-community/UFS_UTILS.git
 [submodule "sorc/verif-global.fd"]
-	path = sorc/verif-global.fd
-	url = https://github.com/NOAA-EMC/EMC_verif-global.git
+  path = sorc/verif-global.fd
+  url = https://github.com/NOAA-EMC/EMC_verif-global.git
 [submodule "sorc/gsi_enkf.fd"]
-	path = sorc/gsi_enkf.fd
-	url = https://github.com/NOAA-EMC/GSI.git
+  path = sorc/gsi_enkf.fd
+  url = https://github.com/NOAA-EMC/GSI.git
 [submodule "sorc/gdas.cd"]
-	path = sorc/gdas.cd
-	url = https://github.com/NOAA-EMC/GDASApp.git
+  path = sorc/gdas.cd
+  url = https://github.com/NOAA-EMC/GDASApp.git
 [submodule "sorc/gsi_utils.fd"]
-	path = sorc/gsi_utils.fd
-	url = https://github.com/NOAA-EMC/GSI-Utils.git
+  path = sorc/gsi_utils.fd
+  url = https://github.com/NOAA-EMC/GSI-Utils.git
 [submodule "sorc/gsi_monitor.fd"]
-	path = sorc/gsi_monitor.fd
-	url = https://github.com/NOAA-EMC/GSI-Monitor.git
+  path = sorc/gsi_monitor.fd
+  url = https://github.com/NOAA-EMC/GSI-Monitor.git

ci/Jenkinsfile

@@ -50,7 +50,7 @@ pipeline {
                             pullRequest.removeLabel("CI-${Machine}-Ready")
                         }
                     }
-                    pullRequest.comment("Building and running on ${Machine} in directory ${HOME}")
+                    echo "Building and running on ${Machine} in directory ${HOME}"
                 }
             }
         }
@@ -76,7 +76,6 @@ pipeline {
                                 ws(HOMEgfs) {
                                     if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { // if the system is already built, skip the build in the case of re-runs
                                         sh(script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim() // TODO: and user configurable control to manage build semphore
-                                        pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}<br>Still doing a checkout to get the latest changes")
                                         checkout scm
                                         dir('sorc') {
                                             sh(script: './link_workflow.sh')
@@ -93,10 +92,14 @@ pipeline {
                                         }
                                     }
                                     if (env.CHANGE_ID && system == 'gfs') {
-                                       if (pullRequest.labels.any { value -> value.matches("CI-${Machine}-Building") }) {
-                                           pullRequest.removeLabel("CI-${Machine}-Building")
-                                       }
-                                       pullRequest.addLabel("CI-${Machine}-Running")
+                                        try {
+                                            if (pullRequest.labels.any { value -> value.matches("CI-${Machine}-Building") }) {
+                                               pullRequest.removeLabel("CI-${Machine}-Building")
+                                            }
+                                            pullRequest.addLabel("CI-${Machine}-Running")
+                                        } catch (Exception e) {
+                                            echo "Failed to update label from Buildng to Running: ${e.getMessage()}"
+                                        }
                                     }
                                     if (system == 'gfs') {
                                         caseList = sh(script: "${HOMEgfs}/ci/scripts/utils/get_host_case_list.py ${machine}", returnStdout: true).trim().split()
@@ -146,11 +149,9 @@ pipeline {
                                 HOMEgfs = "${HOME}/gfs"  // common HOMEgfs is used to launch the scripts that run the experiments
                                 ws(HOMEgfs) {
                                     pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true).trim()
-                                    pullRequest.comment("**Running** experiment: ${Case} on ${Machine}<br>With the experiment in directory:<br>`${HOME}/RUNTESTS/${pslot}`")
                                     try {
                                        sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}")
                                     } catch (Exception e) {
-                                        pullRequest.comment("**FAILURE** running experiment: ${Case} on ${Machine}")
                                         sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_all_batch_jobs ${HOME}/RUNTESTS")
                                         ws(HOME) {
                                             if (fileExists('RUNTESTS/error.logs')) {
@@ -165,7 +166,6 @@ pipeline {
                                         error("Failed to run experiments ${Case} on ${Machine}")
                                     }
                                 }
-                                pullRequest.comment("**SUCCESS** running experiment: ${Case} on ${Machine}")
                             }
                         }
                     }
@@ -179,29 +179,41 @@ pipeline {
         always {
             script {
                 if(env.CHANGE_ID) {
-                    for (label in pullRequest.labels) {
-                        if (label.contains("${Machine}")) {
-                            pullRequest.removeLabel(label)
+                    try {
+                        for (label in pullRequest.labels) {
+                            if (label.contains("${Machine}")) {
+                                pullRequest.removeLabel(label)
+                            }
                         }
+                    }  catch (Exception e) {
+                    echo "Failed to remove labels: ${e.getMessage()}"
                     }
                 }
             }
         }
         success {
             script {
                 if(env.CHANGE_ID) {
-                    pullRequest.addLabel("CI-${Machine}-Passed")
-                    def timestamp = new Date().format('MM dd HH:mm:ss', TimeZone.getTimeZone('America/New_York'))
-                    pullRequest.comment("**CI SUCCESS** ${Machine} at ${timestamp}\n\nBuilt and ran in directory `${HOME}`")
+                    try {
+                        pullRequest.addLabel("CI-${Machine}-Passed")
+                        def timestamp = new Date().format('MM dd HH:mm:ss', TimeZone.getTimeZone('America/New_York'))
+                        pullRequest.comment("**CI SUCCESS** ${Machine} at ${timestamp}\n\nBuilt and ran in directory `${HOME}`")
+                    } catch (Exception e) {
+                        echo "Failed to add success label or comment: ${e.getMessage()}"
+                    }
                 }
             }
         }
         failure {
             script {
                 if(env.CHANGE_ID) {
-                    pullRequest.addLabel("CI-${Machine}-Failed")
-                    def timestamp = new Date().format('MM dd HH:mm:ss', TimeZone.getTimeZone('America/New_York'))
-                    pullRequest.comment("**CI FAILED** ${Machine} at ${timestamp}<br>Built and ran in directory `${HOME}`")
+                    try {
+                        pullRequest.addLabel("CI-${Machine}-Failed")
+                        def timestamp = new Date().format('MM dd HH:mm:ss', TimeZone.getTimeZone('America/New_York'))
+                        pullRequest.comment("**CI FAILED** ${Machine} at ${timestamp}<br>Built and ran in directory `${HOME}`")
+                    } catch (Exception e) {
+                        echo "Failed to add failure label or comment: ${e.getMessage()}"
+                    }
                 }
             }
         }

modulefiles/module_base.hercules.lua

@@ -9,7 +9,6 @@ load(pathJoin("stack-intel", (os.getenv("stack_intel_ver") or "None")))
 load(pathJoin("stack-intel-oneapi-mpi", (os.getenv("stack_impi_ver") or "None")))
 load(pathJoin("intel-oneapi-mkl", (os.getenv("intel_mkl_ver") or "None")))
 load(pathJoin("python", (os.getenv("python_ver") or "None")))
-load(pathJoin("perl", (os.getenv("perl_ver") or "None")))
 
 load(pathJoin("jasper", (os.getenv("jasper_ver") or "None")))
 load(pathJoin("libpng", (os.getenv("libpng_ver") or "None")))

parm/config/gefs/config.base.emc.dyn → parm/config/gefs/config.base

@@ -110,6 +110,7 @@ export RUN="gefs"  # RUN is defined in the job-card (ecf); CDUMP is used at EMC
 # Get all the COM path templates
 source "${EXPDIR}/config.com"
 
+# shellcheck disable=SC2016
 export ERRSCRIPT=${ERRSCRIPT:-'eval [[ $err = 0 ]]'}
 export LOGSCRIPT=${LOGSCRIPT:-""}
 #export ERRSCRIPT=${ERRSCRIPT:-"err_chk"}
@@ -210,7 +211,7 @@ export gfs_cyc=@gfs_cyc@ # 0: no GFS cycle, 1: 00Z only, 2: 00Z and 12Z only, 4:
 export FHMIN_GFS=0
 export FHMIN=${FHMIN_GFS}
 export FHMAX_GFS=@FHMAX_GFS@
-export FHOUT_GFS=6 
+export FHOUT_GFS=6
 export FHMAX_HF_GFS=0
 export FHOUT_HF_GFS=1
 export FHOUT_OCNICE_GFS=6
@@ -259,6 +260,10 @@ export NMEM_ENS=@NMEM_ENS@
 export ENSMEM=${ENSMEM:-"000"}
 export MEMDIR="mem${ENSMEM}"
 
+# initialize ocean ensemble members with perturbations
+#   if true, only occurs for members greater than zero
+export OCN_ENS_PERTURB_FILES=false
+
 export DOIAU="NO"  # While we are not doing IAU, we may want to warm start w/ IAU in the future
 # Check if cycle is cold starting
 if [[ "${EXP_WARM_START}" = ".false." ]]; then

parm/config/gefs/config.stage_ic

@@ -8,6 +8,12 @@ echo "BEGIN: config.stage_ic"
 source "${EXPDIR}/config.resources" stage_ic
 
 case "${CASE}" in
+  "C384")
+    export CPL_ATMIC=""
+    export CPL_ICEIC=""
+    export CPL_OCNIC=""
+    export CPL_WAVIC=""
+    ;;
   "C48")
     export CPL_ATMIC="gefs_test"
     export CPL_ICEIC="gefs_test"

parm/config/gfs/config.base.emc.dyn → parm/config/gfs/config.base

@@ -57,16 +57,9 @@ export DO_NPOESS="NO"      # NPOESS products
 export DO_TRACKER="YES"    # Hurricane track verification
 export DO_GENESIS="YES"    # Cyclone genesis verification
 export DO_GENESIS_FSU="NO" # Cyclone genesis verification (FSU)
-# The monitor is not yet supported on Hercules
-if [[ "${machine}" == "HERCULES" ]]; then
-   export DO_VERFOZN="NO"     # Ozone data assimilation monitoring
-   export DO_VERFRAD="NO"     # Radiance data assimilation monitoring
-   export DO_VMINMON="NO"     # GSI minimization monitoring
-else
-   export DO_VERFOZN="YES"    # Ozone data assimilation monitoring
-   export DO_VERFRAD="YES"    # Radiance data assimilation monitoring
-   export DO_VMINMON="YES"    # GSI minimization monitoring
-fi
+export DO_VERFOZN="YES"    # Ozone data assimilation monitoring
+export DO_VERFRAD="YES"    # Radiance data assimilation monitoring
+export DO_VMINMON="YES"    # GSI minimization monitoring
 export DO_MOS="NO"         # GFS Model Output Statistics - Only supported on WCOSS2
 
 # NO for retrospective parallel; YES for real-time parallel
@@ -132,6 +125,7 @@ export RUN=${RUN:-${CDUMP:-"gfs"}}  # RUN is defined in the job-card (ecf); CDUM
 # Get all the COM path templates
 source "${EXPDIR}/config.com"
 
+# shellcheck disable=SC2016
 export ERRSCRIPT=${ERRSCRIPT:-'eval [[ $err = 0 ]]'}
 export LOGSCRIPT=${LOGSCRIPT:-""}
 #export ERRSCRIPT=${ERRSCRIPT:-"err_chk"}
@@ -247,7 +241,7 @@ export gfs_cyc=@gfs_cyc@ # 0: no GFS cycle, 1: 00Z only, 2: 00Z and 12Z only, 4:
 # GFS output and frequency
 export FHMIN_GFS=0
 export FHMAX_GFS=@FHMAX_GFS@
-export FHOUT_GFS=3 
+export FHOUT_GFS=3
 export FHMAX_HF_GFS=0
 export FHOUT_HF_GFS=1
 export FHOUT_OCNICE_GFS=6

parm/config/gfs/config.prepsnowobs

@@ -8,11 +8,14 @@ echo "BEGIN: config.prepsnowobs"
 # Get task specific resources
 . "${EXPDIR}/config.resources" prepsnowobs
 
-export GTS_OBS_LIST="${HOMEgfs}/sorc/gdas.cd/parm/snow/prep/prep_gts.yaml"
+export GTS_OBS_LIST="${HOMEgfs}/parm/gdas/snow/prep/prep_gts.yaml.j2"
+export IMS_OBS_LIST="${HOMEgfs}/parm/gdas/snow/prep/prep_ims.yaml.j2"
+
 export BUFR2IODAX="${HOMEgfs}/exec/bufr2ioda.x"
-export FIMS_NML_TMPL="${HOMEgfs}/sorc/gdas.cd/parm/snow/prep/fims.nml.j2"
-export IMS_OBS_LIST="${HOMEgfs}/sorc/gdas.cd/parm/snow/prep/prep_ims.yaml"
+
 export CALCFIMSEXE="${HOMEgfs}/exec/calcfIMS.exe"
+export FIMS_NML_TMPL="${HOMEgfs}/parm/gdas/snow/prep/fims.nml.j2"
+
 export IMS2IODACONV="${HOMEgfs}/ush/imsfv3_scf2ioda.py"
 
 echo "END: config.prepsnowobs"

parm/config/gfs/config.snowanl

@@ -6,27 +6,21 @@
 echo "BEGIN: config.snowanl"
 
 # Get task specific resources
-. "${EXPDIR}/config.resources" snowanl
+source "${EXPDIR}/config.resources" snowanl
 
-obs_list_name=gdas_snow_gts_only.yaml
-if [[ "${cyc}" = "18" ]]; then
-    obs_list_name=gdas_snow_prototype.yaml
-fi
-
-export OBS_YAML_DIR=${HOMEgfs}/sorc/gdas.cd/parm/snow/obs/config/
-export OBS_LIST=${HOMEgfs}/sorc/gdas.cd/parm/snow/obs/lists/${obs_list_name}
+export OBS_LIST="${HOMEgfs}/parm/gdas/snow/obs/lists/gdas_snow.yaml.j2"
 
 # Name of the JEDI executable and its yaml template
 export JEDIEXE="${HOMEgfs}/exec/fv3jedi_letkf.x"
-export JEDIYAML="${HOMEgfs}/sorc/gdas.cd/parm/snow/letkfoi/letkfoi.yaml"
+export JEDIYAML="${HOMEgfs}/parm/gdas/snow/letkfoi/letkfoi.yaml.j2"
 
 # Ensemble member properties
 export SNOWDEPTHVAR="snodl"
 export BESTDDEV="30."  # Background Error Std. Dev. for LETKFOI
 
 # Name of the executable that applies increment to bkg and its namelist template
 export APPLY_INCR_EXE="${HOMEgfs}/exec/apply_incr.exe"
-export APPLY_INCR_NML_TMPL="${HOMEgfs}/sorc/gdas.cd/parm/snow/letkfoi/apply_incr_nml.j2"
+export APPLY_INCR_NML_TMPL="${HOMEgfs}/parm/gdas/snow/letkfoi/apply_incr_nml.j2"
 
 export io_layout_x=@IO_LAYOUT_X@
 export io_layout_y=@IO_LAYOUT_Y@

parm/gdas/snow_jedi_fix.yaml.j2

@@ -0,0 +1,7 @@
+mkdir:
+- '{{ DATA }}/fv3jedi'
+copy:
+- ['{{ HOMEgfs }}/fix/gdas/fv3jedi/fv3files/akbk{{ npz }}.nc4', '{{ DATA }}/fv3jedi/akbk.nc4']
+- ['{{ HOMEgfs }}/fix/gdas/fv3jedi/fv3files/fmsmpp.nml', '{{ DATA }}/fv3jedi/fmsmpp.nml']
+- ['{{ HOMEgfs }}/fix/gdas/fv3jedi/fv3files/field_table_gfdl', '{{ DATA }}/fv3jedi/field_table']
+- ['{{ HOMEgfs }}/sorc/gdas.cd/parm/io/fv3jedi_fieldmetadata_restart.yaml', '{{ DATA }}/fv3jedi/fv3jedi_fieldmetadata_restart.yaml']

scripts/exglobal_stage_ic.sh

@@ -104,6 +104,17 @@ for MEMDIR in "${MEMDIR_ARRAY[@]}"; do
         ;;
     esac
 
+    # Ocean Perturbation Files
+    # Extra zero on MEMDIR ensure we have a number even if the string is empty
+    if (( 0${MEMDIR:3} > 0 )) && [[ "${OCN_ENS_PERTURB_FILES:-false}" == "true" ]]; then
+        src="${BASE_CPLIC}/${CPL_OCNIC:-}/${PDY}${cyc}/${MEMDIR}/ocean/${PDY}.${cyc}0000.mom6_increment.nc"
+        tgt="${COM_OCEAN_RESTART_PREV}/${PDY}.${cyc}0000.mom6_increment.nc"
+        ${NCP} "${src}" "${tgt}"
+        rc=${?}
+        ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}"
+        err=$((err + rc))
+    fi
+
     # TODO: Do mediator restarts exists in a ATMW configuration?
     # TODO: No mediator is presumably involved in an ATMA configuration
     if [[ ${EXP_WARM_START:-".false."} = ".true." ]]; then