diff --git a/.gitmodules b/.gitmodules index 5c9e569243..3a95d0ac50 100644 --- a/.gitmodules +++ b/.gitmodules @@ -26,3 +26,6 @@ [submodule "sorc/gsi_monitor.fd"] path = sorc/gsi_monitor.fd url = https://github.com/NOAA-EMC/GSI-Monitor.git +[submodule "sorc/upp.fd"] + path = sorc/upp.fd + url = https://github.com/NOAA-EMC/UPP diff --git a/jobs/rocoto/prepsnowobs.sh b/jobs/rocoto/prepsnowobs.sh index dae11129eb..d260eef46c 100755 --- a/jobs/rocoto/prepsnowobs.sh +++ b/jobs/rocoto/prepsnowobs.sh @@ -14,7 +14,7 @@ export jobid="${job}.$$" ############################################################### # setup python path for workflow utilities and tasks wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src" -gdasappPATH="${HOMEgfs}/sorc/gdas.cd/iodaconv/src:${HOMEgfs}/sorc/gdas.cd/build/lib/python3.7" +gdasappPATH="${HOMEgfs}/sorc/gdas.cd/bundle/iodaconv/src:${HOMEgfs}/sorc/gdas.cd/build/lib/python3.10" PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}:${gdasappPATH}" export PYTHONPATH diff --git a/modulefiles/module_base.hera.lua b/modulefiles/module_base.hera.lua index 8703100e76..1a94019d65 100644 --- a/modulefiles/module_base.hera.lua +++ b/modulefiles/module_base.hera.lua @@ -42,7 +42,7 @@ setenv("WGRIB2","wgrib2") setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) --prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) -prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/feature-GFSv17_com_reorg_log_update/modulefiles")) +prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/dev-gfsv17/modulefiles")) load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None"))) prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) diff --git a/modulefiles/module_gwci.hera.lua b/modulefiles/module_gwci.hera.lua index 3d4c413a44..75b972b69b 100644 --- a/modulefiles/module_gwci.hera.lua +++ b/modulefiles/module_gwci.hera.lua @@ -2,7 +2,7 @@ help([[ Load environment to run GFS workflow setup scripts on Hera ]]) -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev/install/modulefiles/Core") +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev-rocky8/install/modulefiles/Core") load(pathJoin("stack-intel", os.getenv("2021.5.0"))) load(pathJoin("stack-intel-oneapi-mpi", os.getenv("2021.5.1"))) diff --git a/modulefiles/module_gwsetup.hera.lua b/modulefiles/module_gwsetup.hera.lua index 3e8bf2d7f8..696f9577be 100644 --- a/modulefiles/module_gwsetup.hera.lua +++ b/modulefiles/module_gwsetup.hera.lua @@ -4,7 +4,7 @@ Load environment to run GFS workflow setup scripts on Hera load(pathJoin("rocoto")) -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev/install/modulefiles/Core") +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev-rocky8/install/modulefiles/Core") local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" local python_ver=os.getenv("python_ver") or "3.11.6" diff --git a/parm/config/gefs/config.fcst b/parm/config/gefs/config.fcst index 8c3de84357..c3f41c5e7e 100644 --- a/parm/config/gefs/config.fcst +++ b/parm/config/gefs/config.fcst @@ -5,6 +5,8 @@ echo "BEGIN: config.fcst" +export USE_ESMF_THREADING="YES" # Toggle to use ESMF-managed threading or traditional threading in UFSWM + # Turn off waves if not used for this CDUMP case ${WAVE_CDUMP} in both | "${CDUMP/enkf}" ) ;; # Don't change diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources index 1f6485931f..0be7e864a1 100644 --- a/parm/config/gefs/config.resources +++ b/parm/config/gefs/config.resources @@ -101,6 +101,20 @@ case ${step} in ntasks_fv3=${ntasks_fv3_gfs} ntasks_quilt=${ntasks_quilt_gfs} nthreads_fv3=${nthreads_fv3_gfs} + nthreads_ufs=${nthreads_ufs_gfs} + fi + + # Determine if using ESMF-managed threading or traditional threading + # If using traditional threading, set them to 1 + if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then + export UFS_THREADS=1 + else # traditional threading + export UFS_THREADS=${nthreads_ufs:-1} + nthreads_fv3=1 + nthreads_mediator=1 + [[ "${DO_WAVE}" == "YES" ]] && nthreads_ww3=1 + [[ "${DO_OCN}" == "YES" ]] && nthreads_mom6=1 + [[ "${DO_ICE}" == "YES" ]] && nthreads_cice6=1 fi # PETS for the atmosphere dycore @@ -177,11 +191,11 @@ case ${step} in if [[ "${_CDUMP}" =~ "gfs" ]]; then declare -x "npe_${step}_gfs"="${NTASKS_TOT}" - declare -x "nth_${step}_gfs"=1 # ESMF handles threading for the UFS-weather-model + declare -x "nth_${step}_gfs"="${UFS_THREADS}" declare -x "npe_node_${step}_gfs"="${npe_node_max}" else declare -x "npe_${step}"="${NTASKS_TOT}" - declare -x "nth_${step}"=1 # ESMF handles threading for the UFS-weather-model + declare -x "nth_${step}"="${UFS_THREADS}" declare -x "npe_node_${step}"="${npe_node_max}" fi diff --git a/parm/config/gefs/config.ufs b/parm/config/gefs/config.ufs index 17d21573e4..b8695b6dbb 100644 --- a/parm/config/gefs/config.ufs +++ b/parm/config/gefs/config.ufs @@ -78,6 +78,8 @@ case "${fv3_res}" in export layout_y_gfs=1 export nthreads_fv3=1 export nthreads_fv3_gfs=1 + export nthreads_ufs=1 + export nthreads_ufs_gfs=1 export cdmbgwd="0.071,2.1,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="40.0,1.77,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=6.0e-3 # setting for UGWPv1 non-stationary GWD @@ -94,6 +96,8 @@ case "${fv3_res}" in export layout_y_gfs=2 export nthreads_fv3=1 export nthreads_fv3_gfs=1 + export nthreads_ufs=1 + export nthreads_ufs_gfs=1 export cdmbgwd="0.14,1.8,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="20.0,2.5,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=3.0e-3 # setting for UGWPv1 non-stationary GWD @@ -110,6 +114,8 @@ case "${fv3_res}" in export layout_y_gfs=6 export nthreads_fv3=1 export nthreads_fv3_gfs=2 + export nthreads_ufs=1 + export nthreads_ufs_gfs=2 export cdmbgwd="0.23,1.5,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="10.0,3.5,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=1.5e-3 # setting for UGWPv1 non-stationary GWD @@ -126,6 +132,8 @@ case "${fv3_res}" in export layout_y_gfs=8 export nthreads_fv3=1 export nthreads_fv3_gfs=2 + export nthreads_ufs=1 + export nthreads_ufs_gfs=2 export cdmbgwd="1.1,0.72,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="5.0,5.0,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=0.8e-3 # setting for UGWPv1 non-stationary GWD @@ -142,6 +150,8 @@ case "${fv3_res}" in export layout_y_gfs=16 export nthreads_fv3=4 export nthreads_fv3_gfs=4 + export nthreads_ufs=4 + export nthreads_ufs_gfs=4 export cdmbgwd="4.0,0.15,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="2.5,7.5,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=0.5e-3 # setting for UGWPv1 non-stationary GWD @@ -158,6 +168,8 @@ case "${fv3_res}" in export layout_y_gfs=16 export nthreads_fv3=4 export nthreads_fv3_gfs=4 + export nthreads_ufs=4 + export nthreads_ufs_gfs=4 export cdmbgwd="4.0,0.10,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="1.67,8.8,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=0.35e-3 # setting for UGWPv1 non-stationary GWD @@ -174,6 +186,8 @@ case "${fv3_res}" in export layout_y_gfs=32 export nthreads_fv3=4 export nthreads_fv3_gfs=4 + export nthreads_ufs=4 + export nthreads_ufs_gfs=4 export cdmbgwd="4.0,0.05,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="0.625,14.1,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=0.13e-3 # setting for UGWPv1 non-stationary GWD @@ -433,27 +447,30 @@ fi # Set the name of the UFS (previously nems) configure template to use # Default ufs.configure templates for supported model configurations +if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then + tmpl_suffix="_esmf" +fi case "${model_list}" in atm) - default_template="${PARMgfs}/ufs/ufs.configure.atm.IN" + default_template="${PARMgfs}/ufs/ufs.configure.atm${tmpl_suffix:-}.IN" ;; atm.aero) - default_template="${PARMgfs}/ufs/ufs.configure.atmaero.IN" + default_template="${PARMgfs}/ufs/ufs.configure.atmaero${tmpl_suffix:-}.IN" ;; atm.wave) - default_template="${PARMgfs}/ufs/ufs.configure.leapfrog_atm_wav.IN" + default_template="${PARMgfs}/ufs/ufs.configure.leapfrog_atm_wav${tmpl_suffix:-}.IN" ;; atm.ocean.ice) - default_template="${PARMgfs}/ufs/ufs.configure.s2s_esmf.IN" + default_template="${PARMgfs}/ufs/ufs.configure.s2s${tmpl_suffix:-}.IN" ;; atm.ocean.ice.aero) - default_template="${PARMgfs}/ufs/ufs.configure.s2sa_esmf.IN" + default_template="${PARMgfs}/ufs/ufs.configure.s2sa${tmpl_suffix:-}.IN" ;; atm.ocean.ice.wave) - default_template="${PARMgfs}/ufs/ufs.configure.s2sw_esmf.IN" + default_template="${PARMgfs}/ufs/ufs.configure.s2sw${tmpl_suffix:-}.IN" ;; atm.ocean.ice.wave.aero) - default_template="${PARMgfs}/ufs/ufs.configure.s2swa_esmf.IN" + default_template="${PARMgfs}/ufs/ufs.configure.s2swa${tmpl_suffix:-}.IN" ;; *) echo "FATAL ERROR: Unsupported UFSWM configuration for ${model_list}" diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 169fa37430..a0e8531b99 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -142,6 +142,20 @@ export DBNROOT=${DBNROOT:-${UTILROOT:-}/fakedbn} # APP settings export APP=@APP@ +shopt -s extglob +# Adjust APP based on RUN +case "${RUN}" in + gfs) # Turn off aerosols + APP="${APP/%A}" + ;; + enkf*) # Turn off aerosols and waves + APP="${APP/%+([WA])}" + ;; + *) # Keep app unchanged + ;; +esac +shopt -u extglob + # Defaults: export DO_ATM="YES" export DO_COUPLED="NO" @@ -162,6 +176,7 @@ export CASE="@CASECTL@" export CASE_ENS="@CASEENS@" export OCNRES="@OCNRES@" export ICERES="${OCNRES}" + # These are the currently recommended grid-combinations case "${CASE}" in "C48") @@ -383,11 +398,14 @@ export FITSARC="YES" export FHMAX_FITS=132 [[ "${FHMAX_FITS}" -gt "${FHMAX_GFS}" ]] && export FHMAX_FITS=${FHMAX_GFS} -# The monitor jobs are not yet supported for JEDIATMVAR +# The monitor jobs are not yet supported for JEDIATMVAR. if [[ ${DO_JEDIATMVAR} = "YES" ]]; then export DO_VERFOZN="NO" # Ozone data assimilation monitoring export DO_VERFRAD="NO" # Radiance data assimilation monitoring export DO_VMINMON="NO" # GSI minimization monitoring +# The minimization monitor is not yet supported on RDHPCS Hera for Rocky-8 +elif [[ ${machine} = "HERA" ]]; then + export DO_VMINMON="NO" # GSI minimization monitoring fi echo "END: config.base" diff --git a/parm/config/gfs/config.efcs b/parm/config/gfs/config.efcs index effd443730..402ba64087 100644 --- a/parm/config/gfs/config.efcs +++ b/parm/config/gfs/config.efcs @@ -5,12 +5,6 @@ echo "BEGIN: config.efcs" -# Turn off components in ensemble -# export DO_AERO="NO" -# export DO_OCN="NO" -# export DO_ICE="NO" -export DO_WAVE="NO" - export CASE="${CASE_ENS}" # Source model specific information that is resolution dependent diff --git a/parm/config/gfs/config.fcst b/parm/config/gfs/config.fcst index 461f1c5b18..a3e67f8cf9 100644 --- a/parm/config/gfs/config.fcst +++ b/parm/config/gfs/config.fcst @@ -5,6 +5,8 @@ echo "BEGIN: config.fcst" +export USE_ESMF_THREADING="YES" # Toggle to use ESMF-managed threading or traditional threading in UFSWM + # Turn off waves if not used for this CDUMP case ${WAVE_CDUMP} in both | "${CDUMP/enkf}" ) ;; # Don't change diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 21f0a111ef..98fc3b2668 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -570,6 +570,20 @@ case ${step} in ntasks_fv3=${ntasks_fv3_gfs} ntasks_quilt=${ntasks_quilt_gfs} nthreads_fv3=${nthreads_fv3_gfs} + nthreads_ufs=${nthreads_ufs_gfs} + fi + + # Determine if using ESMF-managed threading or traditional threading + # If using traditional threading, set them to 1 + if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then + export UFS_THREADS=1 + else # traditional threading + export UFS_THREADS=${nthreads_ufs:-1} + nthreads_fv3=1 + nthreads_mediator=1 + [[ "${DO_WAVE}" == "YES" ]] && nthreads_ww3=1 + [[ "${DO_OCN}" == "YES" ]] && nthreads_mom6=1 + [[ "${DO_ICE}" == "YES" ]] && nthreads_cice6=1 fi # PETS for the atmosphere dycore @@ -646,11 +660,11 @@ case ${step} in if [[ "${_CDUMP}" =~ "gfs" ]]; then declare -x "npe_${step}_gfs"="${NTASKS_TOT}" - declare -x "nth_${step}_gfs"=1 # ESMF handles threading for the UFS-weather-model + declare -x "nth_${step}_gfs"="${UFS_THREADS}" declare -x "npe_node_${step}_gfs"="${npe_node_max}" else declare -x "npe_${step}"="${NTASKS_TOT}" - declare -x "nth_${step}"=1 # ESMF handles threading for the UFS-weather-model + declare -x "nth_${step}"="${UFS_THREADS}" declare -x "npe_node_${step}"="${npe_node_max}" fi @@ -692,9 +706,14 @@ case ${step} in "C48" | "C96") export npe_upp=${CASE:1} ;; - "C192" | "C384" | "C768") + "C192" | "C384") + export npe_upp=120 + export memory_upp="96GB" + ;; + "C768") export npe_upp=120 - export memory_upp="48GB" + export memory_upp="96GB" + if [[ ${machine} == "WCOSS2" ]]; then export memory_upp="480GB" ; fi ;; *) echo "FATAL ERROR: Resources not defined for job ${job} at resolution ${CASE}" diff --git a/parm/config/gfs/config.ufs b/parm/config/gfs/config.ufs index 68a8941122..0db6f090a5 100644 --- a/parm/config/gfs/config.ufs +++ b/parm/config/gfs/config.ufs @@ -78,6 +78,8 @@ case "${fv3_res}" in export layout_y_gfs=1 export nthreads_fv3=1 export nthreads_fv3_gfs=1 + export nthreads_ufs=1 + export nthreads_ufs_gfs=1 export cdmbgwd="0.071,2.1,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="40.0,1.77,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=6.0e-3 # setting for UGWPv1 non-stationary GWD @@ -94,6 +96,8 @@ case "${fv3_res}" in export layout_y_gfs=2 export nthreads_fv3=1 export nthreads_fv3_gfs=1 + export nthreads_ufs=1 + export nthreads_ufs_gfs=1 export cdmbgwd="0.14,1.8,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="20.0,2.5,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=3.0e-3 # setting for UGWPv1 non-stationary GWD @@ -110,6 +114,8 @@ case "${fv3_res}" in export layout_y_gfs=6 export nthreads_fv3=1 export nthreads_fv3_gfs=2 + export nthreads_ufs=1 + export nthreads_ufs_gfs=2 export cdmbgwd="0.23,1.5,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="10.0,3.5,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=1.5e-3 # setting for UGWPv1 non-stationary GWD @@ -126,6 +132,8 @@ case "${fv3_res}" in export layout_y_gfs=8 export nthreads_fv3=2 export nthreads_fv3_gfs=2 + export nthreads_ufs=2 + export nthreads_ufs_gfs=2 export cdmbgwd="1.1,0.72,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="5.0,5.0,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=0.8e-3 # setting for UGWPv1 non-stationary GWD @@ -142,6 +150,8 @@ case "${fv3_res}" in export layout_y_gfs=16 export nthreads_fv3=4 export nthreads_fv3_gfs=4 + export nthreads_ufs=4 + export nthreads_ufs_gfs=4 export cdmbgwd="4.0,0.15,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="2.5,7.5,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=0.5e-3 # setting for UGWPv1 non-stationary GWD @@ -158,6 +168,8 @@ case "${fv3_res}" in export layout_y_gfs=16 export nthreads_fv3=4 export nthreads_fv3_gfs=4 + export nthreads_ufs=4 + export nthreads_ufs_gfs=4 export cdmbgwd="4.0,0.10,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="1.67,8.8,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=0.35e-3 # setting for UGWPv1 non-stationary GWD @@ -174,6 +186,8 @@ case "${fv3_res}" in export layout_y_gfs=32 export nthreads_fv3=4 export nthreads_fv3_gfs=4 + export nthreads_ufs=4 + export nthreads_ufs_gfs=4 export cdmbgwd="4.0,0.05,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="0.625,14.1,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=0.13e-3 # setting for UGWPv1 non-stationary GWD @@ -420,7 +434,7 @@ if [[ "${skip_ww3}" == "false" ]]; then ntasks_ww3=40 nthreads_ww3=1 ;; - "uglo_m1g16") + "uglo_m1g16") ntasks_ww3=1000 nthreads_ww3=1 ;; @@ -441,27 +455,30 @@ fi # Set the name of the UFS (previously nems) configure template to use # Default ufs.configure templates for supported model configurations +if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then + tmpl_suffix="_esmf" +fi case "${model_list}" in atm) - default_template="${PARMgfs}/ufs/ufs.configure.atm.IN" + default_template="${PARMgfs}/ufs/ufs.configure.atm${tmpl_suffix:-}.IN" ;; atm.aero) - default_template="${PARMgfs}/ufs/ufs.configure.atmaero.IN" + default_template="${PARMgfs}/ufs/ufs.configure.atmaero${tmpl_suffix:-}.IN" ;; atm.wave) - default_template="${PARMgfs}/ufs/ufs.configure.leapfrog_atm_wav.IN" + default_template="${PARMgfs}/ufs/ufs.configure.leapfrog_atm_wav${tmpl_suffix:-}.IN" ;; atm.ocean.ice) - default_template="${PARMgfs}/ufs/ufs.configure.s2s_esmf.IN" + default_template="${PARMgfs}/ufs/ufs.configure.s2s${tmpl_suffix:-}.IN" ;; atm.ocean.ice.aero) - default_template="${PARMgfs}/ufs/ufs.configure.s2sa_esmf.IN" + default_template="${PARMgfs}/ufs/ufs.configure.s2sa${tmpl_suffix:-}.IN" ;; atm.ocean.ice.wave) - default_template="${PARMgfs}/ufs/ufs.configure.s2sw_esmf.IN" + default_template="${PARMgfs}/ufs/ufs.configure.s2sw${tmpl_suffix:-}.IN" ;; atm.ocean.ice.wave.aero) - default_template="${PARMgfs}/ufs/ufs.configure.s2swa_esmf.IN" + default_template="${PARMgfs}/ufs/ufs.configure.s2swa${tmpl_suffix:-}.IN" ;; *) echo "FATAL ERROR: Unsupported UFSWM configuration for ${model_list}" diff --git a/parm/ufs/fv3/diag_table_aod b/parm/ufs/fv3/diag_table_aod index 0de51b66d8..fd8aee1791 100644 --- a/parm/ufs/fv3/diag_table_aod +++ b/parm/ufs/fv3/diag_table_aod @@ -3,4 +3,4 @@ "gfs_phys", "SU_AOD_550", "su_aod550", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "BC_AOD_550", "bc_aod550", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "OC_AOD_550", "oc_aod550", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "SS_AOD_550", "ss_aod550", "fv3_history2d", "all", .false., "none", 2 \ No newline at end of file +"gfs_phys", "SS_AOD_550", "ss_aod550", "fv3_history2d", "all", .false., "none", 2 diff --git a/scripts/exglobal_forecast.sh b/scripts/exglobal_forecast.sh index 0a6329910e..3555d4ef33 100755 --- a/scripts/exglobal_forecast.sh +++ b/scripts/exglobal_forecast.sh @@ -149,6 +149,12 @@ if [[ "${esmf_profile:-}" = ".true." ]]; then export ESMF_RUNTIME_PROFILE_OUTPUT=SUMMARY fi +if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then + unset OMP_NUM_THREADS +else + export OMP_NUM_THREADS=${UFS_THREADS:-1} +fi + ${NCP} "${EXECgfs}/${FCSTEXEC}" "${DATA}/" ${APRUN_UFS} "${DATA}/${FCSTEXEC}" 1>&1 2>&2 export ERR=$? diff --git a/sorc/build_upp.sh b/sorc/build_upp.sh index e217e171db..1dca0035fd 100755 --- a/sorc/build_upp.sh +++ b/sorc/build_upp.sh @@ -26,6 +26,6 @@ if [[ ! -d "../exec" ]]; then mkdir -p ../exec fi -cd ufs_model.fd/FV3/upp/tests +cd upp.fd/tests # shellcheck disable=SC2086 BUILD_JOBS=${BUILD_JOBS:-8} ./compile_upp.sh ${_opts} diff --git a/sorc/gdas.cd b/sorc/gdas.cd index dd350d7e4d..e029bd7609 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit dd350d7e4daab0977407e388711807f13b204f6f +Subproject commit e029bd76095b5604f0d8cceb000f1d7c86e7530f diff --git a/sorc/gfs_utils.fd b/sorc/gfs_utils.fd index 4b7f6095d2..de3708bfb0 160000 --- a/sorc/gfs_utils.fd +++ b/sorc/gfs_utils.fd @@ -1 +1 @@ -Subproject commit 4b7f6095d260b7fcd9c99c337454e170f1aa7f2f +Subproject commit de3708bfb00cd51900e813b84fdf2a3be5d398b0 diff --git a/sorc/gsi_enkf.fd b/sorc/gsi_enkf.fd index 74ac594211..2167bc93fe 160000 --- a/sorc/gsi_enkf.fd +++ b/sorc/gsi_enkf.fd @@ -1 +1 @@ -Subproject commit 74ac5942118d2a83ca84d3a629ec3aaffdb36fc5 +Subproject commit 2167bc93fe2b1e84657822a323666ccc61f9a339 diff --git a/sorc/gsi_monitor.fd b/sorc/gsi_monitor.fd index 149cd811f2..3ecfe07913 160000 --- a/sorc/gsi_monitor.fd +++ b/sorc/gsi_monitor.fd @@ -1 +1 @@ -Subproject commit 149cd811f273883e7d6400eded7b980acd65ca2b +Subproject commit 3ecfe0791391cee11cb0e3a3bcc16957b37057e0 diff --git a/sorc/gsi_utils.fd b/sorc/gsi_utils.fd index 55abe58825..67b014d8d3 160000 --- a/sorc/gsi_utils.fd +++ b/sorc/gsi_utils.fd @@ -1 +1 @@ -Subproject commit 55abe588252ec6f39047d54a14727cf59f7f6688 +Subproject commit 67b014d8d3e5acc1d21aca15e3fe2d66d327a206 diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 2b54f3ea10..336bdb4b06 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -166,12 +166,19 @@ declare -a ufs_templates=("model_configure.IN" \ "MOM6_data_table.IN" \ "ice_in.IN" \ "ufs.configure.atm.IN" \ + "ufs.configure.atm_esmf.IN" \ "ufs.configure.atmaero.IN" \ - "ufs.configure.leapfrog_atm_wav.IN" \ + "ufs.configure.atmaero_esmf.IN" \ + "ufs.configure.s2s.IN" \ "ufs.configure.s2s_esmf.IN" \ + "ufs.configure.s2sa.IN" \ "ufs.configure.s2sa_esmf.IN" \ + "ufs.configure.s2sw.IN" \ "ufs.configure.s2sw_esmf.IN" \ - "ufs.configure.s2swa_esmf.IN" ) + "ufs.configure.s2swa.IN" \ + "ufs.configure.s2swa_esmf.IN" \ + "ufs.configure.leapfrog_atm_wav.IN" \ + "ufs.configure.leapfrog_atm_wav_esmf.IN" ) for file in "${ufs_templates[@]}"; do [[ -s "${file}" ]] && rm -f "${file}" ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/tests/parm/${file}" . @@ -345,10 +352,11 @@ fi #--link source code directories #------------------------------ cd "${HOMEgfs}/sorc" || exit 8 -if [[ -d ufs_model.fd ]]; then - [[ -d upp.fd ]] && rm -rf upp.fd - ${LINK} ufs_model.fd/FV3/upp upp.fd -fi +# TODO: Commenting out until UPP is up-to-date with Rocky-8 in the UFSWM. +#if [[ -d ufs_model.fd ]]; then +# [[ -d upp.fd ]] && rm -rf upp.fd +# ${LINK} ufs_model.fd/FV3/upp upp.fd +#fi if [[ -d gsi_enkf.fd ]]; then [[ -d gsi.fd ]] && rm -rf gsi.fd diff --git a/sorc/ufs_model.fd b/sorc/ufs_model.fd index 6988662728..7fdb58cad0 160000 --- a/sorc/ufs_model.fd +++ b/sorc/ufs_model.fd @@ -1 +1 @@ -Subproject commit 698866272846e8c0f8f61ddb1b20d6463460cd63 +Subproject commit 7fdb58cad0dad2f62ce7813c6719554d1c5a17af diff --git a/sorc/ufs_utils.fd b/sorc/ufs_utils.fd index 1dac85544d..f42fae239d 160000 --- a/sorc/ufs_utils.fd +++ b/sorc/ufs_utils.fd @@ -1 +1 @@ -Subproject commit 1dac85544dd4134de7cb899214c7db67bbe8c0b4 +Subproject commit f42fae239d0824f7b9a83c9afdc3d980894c7df8 diff --git a/sorc/upp.fd b/sorc/upp.fd new file mode 160000 index 0000000000..e2189f7d0d --- /dev/null +++ b/sorc/upp.fd @@ -0,0 +1 @@ +Subproject commit e2189f7d0dcaa3ce15050618a3eeb2c4ef8774be diff --git a/sorc/verif-global.fd b/sorc/verif-global.fd index bd1c8f62a1..9377e84ba3 160000 --- a/sorc/verif-global.fd +++ b/sorc/verif-global.fd @@ -1 +1 @@ -Subproject commit bd1c8f62a1878051e34ff7c6f6a4dd290381f1ef +Subproject commit 9377e84ba3fc9b2fd13c2c84cfd571855dee75ae diff --git a/ush/hpssarch_gen.sh b/ush/hpssarch_gen.sh index eadd465421..8b61cac948 100755 --- a/ush/hpssarch_gen.sh +++ b/ush/hpssarch_gen.sh @@ -681,7 +681,7 @@ if [[ ${type} == "enkfgdas" || ${type} == "enkfgfs" ]]; then touch "${DATA}/${RUN}_restartb_grp${n}.txt" m=1 - while (( m <= NMEM_EARCGRP )); do + while (( m <= NMEM_EARCGRP && (n-1)*NMEM_EARCGRP+m <= NMEM_ENS )); do nm=$(((n-1)*NMEM_EARCGRP+m)) mem=$(printf %03i ${nm}) head="${RUN}.t${cyc}z." diff --git a/ush/load_ufsda_modules.sh b/ush/load_ufsda_modules.sh index e8e72b8fbe..6bab04b3bf 100755 --- a/ush/load_ufsda_modules.sh +++ b/ush/load_ufsda_modules.sh @@ -42,14 +42,14 @@ case "${MACHINE_ID}" in ;; ("wcoss2" | "acorn" | "jet" | "gaea" | "s4") echo WARNING: UFSDA NOT SUPPORTED ON THIS PLATFORM - ;; + ;; *) echo "WARNING: UNKNOWN PLATFORM" ;; esac module list -pip list +#pip list # Restore stack soft limit: ulimit -S -s "${ulimit_s}" diff --git a/ush/module-setup.sh b/ush/module-setup.sh index 008ef7ee97..b66e3622d0 100755 --- a/ush/module-setup.sh +++ b/ush/module-setup.sh @@ -36,10 +36,10 @@ elif [[ ${MACHINE_ID} = orion* ]] ; then if ( ! eval module help > /dev/null 2>&1 ) ; then source /apps/lmod/lmod/init/bash fi - export LMOD_SYSTEM_DEFAULT_MODULES=contrib - set +u - module reset - set -u + #export LMOD_SYSTEM_DEFAULT_MODULES=git/2.28.0 # contrib has a lot of stuff we shouldn't put in MODULEPATH + #set +u + module purge # reset causes issues on Orion sometimes. + #set -u elif [[ ${MACHINE_ID} = s4* ]] ; then # We are on SSEC Wisconsin S4 diff --git a/versions/build.hera.ver b/versions/build.hera.ver index 263e967a97..337d5c32da 100644 --- a/versions/build.hera.ver +++ b/versions/build.hera.ver @@ -1,5 +1,5 @@ export stack_intel_ver=2021.5.0 export stack_impi_ver=2021.5.1 -export spack_env=gsi-addon-dev +export spack_env=gsi-addon-dev-rocky8 source "${HOMEgfs:-}/versions/build.spack.ver" export spack_mod_path="/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core" diff --git a/versions/run.hera.ver b/versions/run.hera.ver index 4529d34821..b358f9d495 100644 --- a/versions/run.hera.ver +++ b/versions/run.hera.ver @@ -1,6 +1,6 @@ export stack_intel_ver=2021.5.0 export stack_impi_ver=2021.5.1 -export spack_env=gsi-addon-dev +export spack_env=gsi-addon-dev-rocky8 export hpss_ver=hpss export ncl_ver=6.6.2 diff --git a/versions/run.spack.ver b/versions/run.spack.ver index c00b7483cd..412e8c93ea 100644 --- a/versions/run.spack.ver +++ b/versions/run.spack.ver @@ -31,4 +31,4 @@ export obsproc_run_ver=1.1.2 export prepobs_run_ver=1.0.1 export ens_tracker_ver=feature-GFSv17_com_reorg -export fit2obs_ver=1.0.0 +export fit2obs_ver=1.1.0 diff --git a/workflow/setup_expt.py b/workflow/setup_expt.py index 90e890910f..c689ec2ae7 100755 --- a/workflow/setup_expt.py +++ b/workflow/setup_expt.py @@ -385,7 +385,7 @@ def input_args(*argv): Method to collect user arguments for `setup_expt.py` """ - ufs_apps = ['ATM', 'ATMA', 'ATMW', 'S2S', 'S2SA', 'S2SW'] + ufs_apps = ['ATM', 'ATMA', 'ATMW', 'S2S', 'S2SA', 'S2SW', 'S2SWA'] def _common_args(parser): parser.add_argument('--pslot', help='parallel experiment name', @@ -433,7 +433,7 @@ def _gfs_or_gefs_ensemble_args(parser): def _gfs_or_gefs_forecast_args(parser): parser.add_argument('--app', help='UFS application', type=str, - choices=ufs_apps + ['S2SWA'], required=False, default='ATM') + choices=ufs_apps, required=False, default='ATM') parser.add_argument('--gfs_cyc', help='Number of forecasts per day', type=int, choices=[1, 2, 4], default=1, required=False) return parser