Combining PRs for streamlining testing and merging

.gitmodules

@@ -26,3 +26,6 @@
 [submodule "sorc/gsi_monitor.fd"]
   path = sorc/gsi_monitor.fd
   url = https://github.com/NOAA-EMC/GSI-Monitor.git
+[submodule "sorc/upp.fd"]
+  path = sorc/upp.fd
+  url = https://github.com/NOAA-EMC/UPP

jobs/rocoto/prepsnowobs.sh

@@ -14,7 +14,7 @@ export jobid="${job}.$$"
 ###############################################################
 # setup python path for workflow utilities and tasks
 wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-gdasappPATH="${HOMEgfs}/sorc/gdas.cd/iodaconv/src:${HOMEgfs}/sorc/gdas.cd/build/lib/python3.7"
+gdasappPATH="${HOMEgfs}/sorc/gdas.cd/bundle/iodaconv/src:${HOMEgfs}/sorc/gdas.cd/build/lib/python3.10"
 PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}:${gdasappPATH}"
 export PYTHONPATH
 

modulefiles/module_base.hera.lua

@@ -42,7 +42,7 @@ setenv("WGRIB2","wgrib2")
 setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None"))
 
 --prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles"))
-prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/feature-GFSv17_com_reorg_log_update/modulefiles"))
+prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/dev-gfsv17/modulefiles"))
 load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None")))
 
 prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles"))

modulefiles/module_gwci.hera.lua

@@ -2,7 +2,7 @@ help([[
 Load environment to run GFS workflow setup scripts on Hera
 ]])
 
-prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev/install/modulefiles/Core")
+prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev-rocky8/install/modulefiles/Core")
 
 load(pathJoin("stack-intel", os.getenv("2021.5.0")))
 load(pathJoin("stack-intel-oneapi-mpi", os.getenv("2021.5.1")))

modulefiles/module_gwsetup.hera.lua

@@ -4,7 +4,7 @@ Load environment to run GFS workflow setup scripts on Hera
 
 load(pathJoin("rocoto"))
 
-prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev/install/modulefiles/Core")
+prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev-rocky8/install/modulefiles/Core")
 
 local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0"
 local python_ver=os.getenv("python_ver") or "3.11.6"

parm/config/gefs/config.fcst

@@ -5,6 +5,8 @@
 
 echo "BEGIN: config.fcst"
 
+export USE_ESMF_THREADING="YES"  # Toggle to use ESMF-managed threading or traditional threading in UFSWM
+
 # Turn off waves if not used for this CDUMP
 case ${WAVE_CDUMP} in
   both | "${CDUMP/enkf}" ) ;; # Don't change

parm/config/gefs/config.resources

@@ -101,6 +101,20 @@ case ${step} in
         ntasks_fv3=${ntasks_fv3_gfs}
         ntasks_quilt=${ntasks_quilt_gfs}
         nthreads_fv3=${nthreads_fv3_gfs}
+        nthreads_ufs=${nthreads_ufs_gfs}
+      fi
+
+      # Determine if using ESMF-managed threading or traditional threading
+      # If using traditional threading, set them to 1
+      if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then
+        export UFS_THREADS=1
+      else  # traditional threading
+        export UFS_THREADS=${nthreads_ufs:-1}
+        nthreads_fv3=1
+        nthreads_mediator=1
+        [[ "${DO_WAVE}" == "YES" ]] && nthreads_ww3=1
+        [[ "${DO_OCN}" == "YES" ]] && nthreads_mom6=1
+        [[ "${DO_ICE}" == "YES" ]] && nthreads_cice6=1
       fi
 
       # PETS for the atmosphere dycore
@@ -177,11 +191,11 @@ case ${step} in
 
       if [[ "${_CDUMP}" =~ "gfs" ]]; then
         declare -x "npe_${step}_gfs"="${NTASKS_TOT}"
-        declare -x "nth_${step}_gfs"=1  # ESMF handles threading for the UFS-weather-model
+        declare -x "nth_${step}_gfs"="${UFS_THREADS}"
         declare -x "npe_node_${step}_gfs"="${npe_node_max}"
       else
         declare -x "npe_${step}"="${NTASKS_TOT}"
-        declare -x "nth_${step}"=1  # ESMF handles threading for the UFS-weather-model
+        declare -x "nth_${step}"="${UFS_THREADS}"
         declare -x "npe_node_${step}"="${npe_node_max}"
       fi
 

parm/config/gefs/config.ufs

@@ -78,6 +78,8 @@ case "${fv3_res}" in
         export layout_y_gfs=1
         export nthreads_fv3=1
         export nthreads_fv3_gfs=1
+        export nthreads_ufs=1
+        export nthreads_ufs_gfs=1
         export cdmbgwd="0.071,2.1,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
         export cdmbgwd_gsl="40.0,1.77,1.0,1.0"   # settings for GSL drag suite
         export knob_ugwp_tauamp=6.0e-3      # setting for UGWPv1 non-stationary GWD
@@ -94,6 +96,8 @@ case "${fv3_res}" in
         export layout_y_gfs=2
         export nthreads_fv3=1
         export nthreads_fv3_gfs=1
+        export nthreads_ufs=1
+        export nthreads_ufs_gfs=1
         export cdmbgwd="0.14,1.8,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
         export cdmbgwd_gsl="20.0,2.5,1.0,1.0"   # settings for GSL drag suite
         export knob_ugwp_tauamp=3.0e-3      # setting for UGWPv1 non-stationary GWD
@@ -110,6 +114,8 @@ case "${fv3_res}" in
         export layout_y_gfs=6
         export nthreads_fv3=1
         export nthreads_fv3_gfs=2
+        export nthreads_ufs=1
+        export nthreads_ufs_gfs=2
         export cdmbgwd="0.23,1.5,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
         export cdmbgwd_gsl="10.0,3.5,1.0,1.0"   # settings for GSL drag suite
         export knob_ugwp_tauamp=1.5e-3      # setting for UGWPv1 non-stationary GWD
@@ -126,6 +132,8 @@ case "${fv3_res}" in
         export layout_y_gfs=8
         export nthreads_fv3=1
         export nthreads_fv3_gfs=2
+        export nthreads_ufs=1
+        export nthreads_ufs_gfs=2
         export cdmbgwd="1.1,0.72,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
         export cdmbgwd_gsl="5.0,5.0,1.0,1.0"   # settings for GSL drag suite
         export knob_ugwp_tauamp=0.8e-3      # setting for UGWPv1 non-stationary GWD
@@ -142,6 +150,8 @@ case "${fv3_res}" in
         export layout_y_gfs=16
         export nthreads_fv3=4
         export nthreads_fv3_gfs=4
+        export nthreads_ufs=4
+        export nthreads_ufs_gfs=4
         export cdmbgwd="4.0,0.15,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
         export cdmbgwd_gsl="2.5,7.5,1.0,1.0"   # settings for GSL drag suite
         export knob_ugwp_tauamp=0.5e-3      # setting for UGWPv1 non-stationary GWD
@@ -158,6 +168,8 @@ case "${fv3_res}" in
         export layout_y_gfs=16
         export nthreads_fv3=4
         export nthreads_fv3_gfs=4
+        export nthreads_ufs=4
+        export nthreads_ufs_gfs=4
         export cdmbgwd="4.0,0.10,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
         export cdmbgwd_gsl="1.67,8.8,1.0,1.0"   # settings for GSL drag suite
         export knob_ugwp_tauamp=0.35e-3      # setting for UGWPv1 non-stationary GWD
@@ -174,6 +186,8 @@ case "${fv3_res}" in
         export layout_y_gfs=32
         export nthreads_fv3=4
         export nthreads_fv3_gfs=4
+        export nthreads_ufs=4
+        export nthreads_ufs_gfs=4
         export cdmbgwd="4.0,0.05,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
         export cdmbgwd_gsl="0.625,14.1,1.0,1.0"   # settings for GSL drag suite
         export knob_ugwp_tauamp=0.13e-3      # setting for UGWPv1 non-stationary GWD
@@ -433,27 +447,30 @@ fi
 
 # Set the name of the UFS (previously nems) configure template to use
 # Default ufs.configure templates for supported model configurations
+if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then
+  tmpl_suffix="_esmf"
+fi
 case "${model_list}" in
   atm)
-    default_template="${PARMgfs}/ufs/ufs.configure.atm.IN"
+    default_template="${PARMgfs}/ufs/ufs.configure.atm${tmpl_suffix:-}.IN"
     ;;
   atm.aero)
-    default_template="${PARMgfs}/ufs/ufs.configure.atmaero.IN"
+    default_template="${PARMgfs}/ufs/ufs.configure.atmaero${tmpl_suffix:-}.IN"
     ;;
   atm.wave)
-    default_template="${PARMgfs}/ufs/ufs.configure.leapfrog_atm_wav.IN"
+    default_template="${PARMgfs}/ufs/ufs.configure.leapfrog_atm_wav${tmpl_suffix:-}.IN"
     ;;
   atm.ocean.ice)
-    default_template="${PARMgfs}/ufs/ufs.configure.s2s_esmf.IN"
+    default_template="${PARMgfs}/ufs/ufs.configure.s2s${tmpl_suffix:-}.IN"
     ;;
   atm.ocean.ice.aero)
-    default_template="${PARMgfs}/ufs/ufs.configure.s2sa_esmf.IN"
+    default_template="${PARMgfs}/ufs/ufs.configure.s2sa${tmpl_suffix:-}.IN"
     ;;
   atm.ocean.ice.wave)
-    default_template="${PARMgfs}/ufs/ufs.configure.s2sw_esmf.IN"
+    default_template="${PARMgfs}/ufs/ufs.configure.s2sw${tmpl_suffix:-}.IN"
     ;;
   atm.ocean.ice.wave.aero)
-    default_template="${PARMgfs}/ufs/ufs.configure.s2swa_esmf.IN"
+    default_template="${PARMgfs}/ufs/ufs.configure.s2swa${tmpl_suffix:-}.IN"
     ;;
   *)
     echo "FATAL ERROR: Unsupported UFSWM configuration for ${model_list}"

parm/config/gfs/config.base

@@ -142,6 +142,20 @@ export DBNROOT=${DBNROOT:-${UTILROOT:-}/fakedbn}
 # APP settings
 export APP=@APP@
 
+shopt -s extglob
+# Adjust APP based on RUN
+case "${RUN}" in
+    gfs)   # Turn off aerosols
+        APP="${APP/%A}"
+        ;;
+    enkf*) # Turn off aerosols and waves
+        APP="${APP/%+([WA])}"
+        ;;
+    *)     # Keep app unchanged
+        ;;
+esac
+shopt -u extglob
+
 # Defaults:
 export DO_ATM="YES"
 export DO_COUPLED="NO"
@@ -162,6 +176,7 @@ export CASE="@CASECTL@"
 export CASE_ENS="@CASEENS@"
 export OCNRES="@OCNRES@"
 export ICERES="${OCNRES}"
+
 # These are the currently recommended grid-combinations
 case "${CASE}" in
     "C48")
@@ -383,11 +398,14 @@ export FITSARC="YES"
 export FHMAX_FITS=132
 [[ "${FHMAX_FITS}" -gt "${FHMAX_GFS}" ]] && export FHMAX_FITS=${FHMAX_GFS}
 
-# The monitor jobs are not yet supported for JEDIATMVAR
+# The monitor jobs are not yet supported for JEDIATMVAR.
 if [[ ${DO_JEDIATMVAR} = "YES" ]]; then
    export DO_VERFOZN="NO"     # Ozone data assimilation monitoring
    export DO_VERFRAD="NO"     # Radiance data assimilation monitoring
    export DO_VMINMON="NO"     # GSI minimization monitoring
+# The minimization monitor is not yet supported on RDHPCS Hera for Rocky-8
+elif [[ ${machine} = "HERA" ]]; then
+   export DO_VMINMON="NO"     # GSI minimization monitoring
 fi
 
 echo "END: config.base"

parm/config/gfs/config.efcs

@@ -5,12 +5,6 @@
 
 echo "BEGIN: config.efcs"
 
-# Turn off components in ensemble
-# export DO_AERO="NO"
-# export DO_OCN="NO"
-# export DO_ICE="NO"
-export DO_WAVE="NO"
-
 export CASE="${CASE_ENS}"
 
 # Source model specific information that is resolution dependent

parm/config/gfs/config.fcst

@@ -5,6 +5,8 @@
 
 echo "BEGIN: config.fcst"
 
+export USE_ESMF_THREADING="YES"  # Toggle to use ESMF-managed threading or traditional threading in UFSWM
+
 # Turn off waves if not used for this CDUMP
 case ${WAVE_CDUMP} in
   both | "${CDUMP/enkf}" ) ;; # Don't change

parm/config/gfs/config.resources

@@ -570,6 +570,20 @@ case ${step} in
         ntasks_fv3=${ntasks_fv3_gfs}
         ntasks_quilt=${ntasks_quilt_gfs}
         nthreads_fv3=${nthreads_fv3_gfs}
+        nthreads_ufs=${nthreads_ufs_gfs}
+      fi
+
+      # Determine if using ESMF-managed threading or traditional threading
+      # If using traditional threading, set them to 1
+      if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then
+        export UFS_THREADS=1
+      else  # traditional threading
+        export UFS_THREADS=${nthreads_ufs:-1}
+        nthreads_fv3=1
+        nthreads_mediator=1
+        [[ "${DO_WAVE}" == "YES" ]] && nthreads_ww3=1
+        [[ "${DO_OCN}" == "YES" ]] && nthreads_mom6=1
+        [[ "${DO_ICE}" == "YES" ]] && nthreads_cice6=1
       fi
 
       # PETS for the atmosphere dycore
@@ -646,11 +660,11 @@ case ${step} in
 
       if [[ "${_CDUMP}" =~ "gfs" ]]; then
         declare -x "npe_${step}_gfs"="${NTASKS_TOT}"
-        declare -x "nth_${step}_gfs"=1  # ESMF handles threading for the UFS-weather-model
+        declare -x "nth_${step}_gfs"="${UFS_THREADS}"
         declare -x "npe_node_${step}_gfs"="${npe_node_max}"
       else
         declare -x "npe_${step}"="${NTASKS_TOT}"
-        declare -x "nth_${step}"=1  # ESMF handles threading for the UFS-weather-model
+        declare -x "nth_${step}"="${UFS_THREADS}"
         declare -x "npe_node_${step}"="${npe_node_max}"
       fi
 
@@ -692,9 +706,14 @@ case ${step} in
       "C48" | "C96")
         export npe_upp=${CASE:1}
       ;;
-      "C192" | "C384" | "C768")
+      "C192" | "C384")
+        export npe_upp=120
+        export memory_upp="96GB"
+      ;;
+      "C768")
         export npe_upp=120
-        export memory_upp="48GB"
+        export memory_upp="96GB"
+        if [[ ${machine} == "WCOSS2" ]]; then export memory_upp="480GB" ; fi
       ;;
       *)
         echo "FATAL ERROR: Resources not defined for job ${job} at resolution ${CASE}"