diff --git a/parm/config/gefs/config.fcst b/parm/config/gefs/config.fcst index 8c3de84357..c3f41c5e7e 100644 --- a/parm/config/gefs/config.fcst +++ b/parm/config/gefs/config.fcst @@ -5,6 +5,8 @@ echo "BEGIN: config.fcst" +export USE_ESMF_THREADING="YES" # Toggle to use ESMF-managed threading or traditional threading in UFSWM + # Turn off waves if not used for this CDUMP case ${WAVE_CDUMP} in both | "${CDUMP/enkf}" ) ;; # Don't change diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources index 1f6485931f..0be7e864a1 100644 --- a/parm/config/gefs/config.resources +++ b/parm/config/gefs/config.resources @@ -101,6 +101,20 @@ case ${step} in ntasks_fv3=${ntasks_fv3_gfs} ntasks_quilt=${ntasks_quilt_gfs} nthreads_fv3=${nthreads_fv3_gfs} + nthreads_ufs=${nthreads_ufs_gfs} + fi + + # Determine if using ESMF-managed threading or traditional threading + # If using traditional threading, set them to 1 + if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then + export UFS_THREADS=1 + else # traditional threading + export UFS_THREADS=${nthreads_ufs:-1} + nthreads_fv3=1 + nthreads_mediator=1 + [[ "${DO_WAVE}" == "YES" ]] && nthreads_ww3=1 + [[ "${DO_OCN}" == "YES" ]] && nthreads_mom6=1 + [[ "${DO_ICE}" == "YES" ]] && nthreads_cice6=1 fi # PETS for the atmosphere dycore @@ -177,11 +191,11 @@ case ${step} in if [[ "${_CDUMP}" =~ "gfs" ]]; then declare -x "npe_${step}_gfs"="${NTASKS_TOT}" - declare -x "nth_${step}_gfs"=1 # ESMF handles threading for the UFS-weather-model + declare -x "nth_${step}_gfs"="${UFS_THREADS}" declare -x "npe_node_${step}_gfs"="${npe_node_max}" else declare -x "npe_${step}"="${NTASKS_TOT}" - declare -x "nth_${step}"=1 # ESMF handles threading for the UFS-weather-model + declare -x "nth_${step}"="${UFS_THREADS}" declare -x "npe_node_${step}"="${npe_node_max}" fi diff --git a/parm/config/gefs/config.ufs b/parm/config/gefs/config.ufs index 17d21573e4..b8695b6dbb 100644 --- a/parm/config/gefs/config.ufs +++ b/parm/config/gefs/config.ufs @@ -78,6 +78,8 @@ case "${fv3_res}" in export layout_y_gfs=1 export nthreads_fv3=1 export nthreads_fv3_gfs=1 + export nthreads_ufs=1 + export nthreads_ufs_gfs=1 export cdmbgwd="0.071,2.1,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="40.0,1.77,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=6.0e-3 # setting for UGWPv1 non-stationary GWD @@ -94,6 +96,8 @@ case "${fv3_res}" in export layout_y_gfs=2 export nthreads_fv3=1 export nthreads_fv3_gfs=1 + export nthreads_ufs=1 + export nthreads_ufs_gfs=1 export cdmbgwd="0.14,1.8,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="20.0,2.5,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=3.0e-3 # setting for UGWPv1 non-stationary GWD @@ -110,6 +114,8 @@ case "${fv3_res}" in export layout_y_gfs=6 export nthreads_fv3=1 export nthreads_fv3_gfs=2 + export nthreads_ufs=1 + export nthreads_ufs_gfs=2 export cdmbgwd="0.23,1.5,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="10.0,3.5,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=1.5e-3 # setting for UGWPv1 non-stationary GWD @@ -126,6 +132,8 @@ case "${fv3_res}" in export layout_y_gfs=8 export nthreads_fv3=1 export nthreads_fv3_gfs=2 + export nthreads_ufs=1 + export nthreads_ufs_gfs=2 export cdmbgwd="1.1,0.72,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="5.0,5.0,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=0.8e-3 # setting for UGWPv1 non-stationary GWD @@ -142,6 +150,8 @@ case "${fv3_res}" in export layout_y_gfs=16 export nthreads_fv3=4 export nthreads_fv3_gfs=4 + export nthreads_ufs=4 + export nthreads_ufs_gfs=4 export cdmbgwd="4.0,0.15,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="2.5,7.5,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=0.5e-3 # setting for UGWPv1 non-stationary GWD @@ -158,6 +168,8 @@ case "${fv3_res}" in export layout_y_gfs=16 export nthreads_fv3=4 export nthreads_fv3_gfs=4 + export nthreads_ufs=4 + export nthreads_ufs_gfs=4 export cdmbgwd="4.0,0.10,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="1.67,8.8,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=0.35e-3 # setting for UGWPv1 non-stationary GWD @@ -174,6 +186,8 @@ case "${fv3_res}" in export layout_y_gfs=32 export nthreads_fv3=4 export nthreads_fv3_gfs=4 + export nthreads_ufs=4 + export nthreads_ufs_gfs=4 export cdmbgwd="4.0,0.05,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="0.625,14.1,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=0.13e-3 # setting for UGWPv1 non-stationary GWD @@ -433,27 +447,30 @@ fi # Set the name of the UFS (previously nems) configure template to use # Default ufs.configure templates for supported model configurations +if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then + tmpl_suffix="_esmf" +fi case "${model_list}" in atm) - default_template="${PARMgfs}/ufs/ufs.configure.atm.IN" + default_template="${PARMgfs}/ufs/ufs.configure.atm${tmpl_suffix:-}.IN" ;; atm.aero) - default_template="${PARMgfs}/ufs/ufs.configure.atmaero.IN" + default_template="${PARMgfs}/ufs/ufs.configure.atmaero${tmpl_suffix:-}.IN" ;; atm.wave) - default_template="${PARMgfs}/ufs/ufs.configure.leapfrog_atm_wav.IN" + default_template="${PARMgfs}/ufs/ufs.configure.leapfrog_atm_wav${tmpl_suffix:-}.IN" ;; atm.ocean.ice) - default_template="${PARMgfs}/ufs/ufs.configure.s2s_esmf.IN" + default_template="${PARMgfs}/ufs/ufs.configure.s2s${tmpl_suffix:-}.IN" ;; atm.ocean.ice.aero) - default_template="${PARMgfs}/ufs/ufs.configure.s2sa_esmf.IN" + default_template="${PARMgfs}/ufs/ufs.configure.s2sa${tmpl_suffix:-}.IN" ;; atm.ocean.ice.wave) - default_template="${PARMgfs}/ufs/ufs.configure.s2sw_esmf.IN" + default_template="${PARMgfs}/ufs/ufs.configure.s2sw${tmpl_suffix:-}.IN" ;; atm.ocean.ice.wave.aero) - default_template="${PARMgfs}/ufs/ufs.configure.s2swa_esmf.IN" + default_template="${PARMgfs}/ufs/ufs.configure.s2swa${tmpl_suffix:-}.IN" ;; *) echo "FATAL ERROR: Unsupported UFSWM configuration for ${model_list}" diff --git a/parm/config/gfs/config.fcst b/parm/config/gfs/config.fcst index 461f1c5b18..a3e67f8cf9 100644 --- a/parm/config/gfs/config.fcst +++ b/parm/config/gfs/config.fcst @@ -5,6 +5,8 @@ echo "BEGIN: config.fcst" +export USE_ESMF_THREADING="YES" # Toggle to use ESMF-managed threading or traditional threading in UFSWM + # Turn off waves if not used for this CDUMP case ${WAVE_CDUMP} in both | "${CDUMP/enkf}" ) ;; # Don't change diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 21f0a111ef..98fc3b2668 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -570,6 +570,20 @@ case ${step} in ntasks_fv3=${ntasks_fv3_gfs} ntasks_quilt=${ntasks_quilt_gfs} nthreads_fv3=${nthreads_fv3_gfs} + nthreads_ufs=${nthreads_ufs_gfs} + fi + + # Determine if using ESMF-managed threading or traditional threading + # If using traditional threading, set them to 1 + if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then + export UFS_THREADS=1 + else # traditional threading + export UFS_THREADS=${nthreads_ufs:-1} + nthreads_fv3=1 + nthreads_mediator=1 + [[ "${DO_WAVE}" == "YES" ]] && nthreads_ww3=1 + [[ "${DO_OCN}" == "YES" ]] && nthreads_mom6=1 + [[ "${DO_ICE}" == "YES" ]] && nthreads_cice6=1 fi # PETS for the atmosphere dycore @@ -646,11 +660,11 @@ case ${step} in if [[ "${_CDUMP}" =~ "gfs" ]]; then declare -x "npe_${step}_gfs"="${NTASKS_TOT}" - declare -x "nth_${step}_gfs"=1 # ESMF handles threading for the UFS-weather-model + declare -x "nth_${step}_gfs"="${UFS_THREADS}" declare -x "npe_node_${step}_gfs"="${npe_node_max}" else declare -x "npe_${step}"="${NTASKS_TOT}" - declare -x "nth_${step}"=1 # ESMF handles threading for the UFS-weather-model + declare -x "nth_${step}"="${UFS_THREADS}" declare -x "npe_node_${step}"="${npe_node_max}" fi @@ -692,9 +706,14 @@ case ${step} in "C48" | "C96") export npe_upp=${CASE:1} ;; - "C192" | "C384" | "C768") + "C192" | "C384") + export npe_upp=120 + export memory_upp="96GB" + ;; + "C768") export npe_upp=120 - export memory_upp="48GB" + export memory_upp="96GB" + if [[ ${machine} == "WCOSS2" ]]; then export memory_upp="480GB" ; fi ;; *) echo "FATAL ERROR: Resources not defined for job ${job} at resolution ${CASE}" diff --git a/parm/config/gfs/config.ufs b/parm/config/gfs/config.ufs index 68a8941122..0db6f090a5 100644 --- a/parm/config/gfs/config.ufs +++ b/parm/config/gfs/config.ufs @@ -78,6 +78,8 @@ case "${fv3_res}" in export layout_y_gfs=1 export nthreads_fv3=1 export nthreads_fv3_gfs=1 + export nthreads_ufs=1 + export nthreads_ufs_gfs=1 export cdmbgwd="0.071,2.1,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="40.0,1.77,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=6.0e-3 # setting for UGWPv1 non-stationary GWD @@ -94,6 +96,8 @@ case "${fv3_res}" in export layout_y_gfs=2 export nthreads_fv3=1 export nthreads_fv3_gfs=1 + export nthreads_ufs=1 + export nthreads_ufs_gfs=1 export cdmbgwd="0.14,1.8,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="20.0,2.5,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=3.0e-3 # setting for UGWPv1 non-stationary GWD @@ -110,6 +114,8 @@ case "${fv3_res}" in export layout_y_gfs=6 export nthreads_fv3=1 export nthreads_fv3_gfs=2 + export nthreads_ufs=1 + export nthreads_ufs_gfs=2 export cdmbgwd="0.23,1.5,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="10.0,3.5,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=1.5e-3 # setting for UGWPv1 non-stationary GWD @@ -126,6 +132,8 @@ case "${fv3_res}" in export layout_y_gfs=8 export nthreads_fv3=2 export nthreads_fv3_gfs=2 + export nthreads_ufs=2 + export nthreads_ufs_gfs=2 export cdmbgwd="1.1,0.72,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="5.0,5.0,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=0.8e-3 # setting for UGWPv1 non-stationary GWD @@ -142,6 +150,8 @@ case "${fv3_res}" in export layout_y_gfs=16 export nthreads_fv3=4 export nthreads_fv3_gfs=4 + export nthreads_ufs=4 + export nthreads_ufs_gfs=4 export cdmbgwd="4.0,0.15,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="2.5,7.5,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=0.5e-3 # setting for UGWPv1 non-stationary GWD @@ -158,6 +168,8 @@ case "${fv3_res}" in export layout_y_gfs=16 export nthreads_fv3=4 export nthreads_fv3_gfs=4 + export nthreads_ufs=4 + export nthreads_ufs_gfs=4 export cdmbgwd="4.0,0.10,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="1.67,8.8,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=0.35e-3 # setting for UGWPv1 non-stationary GWD @@ -174,6 +186,8 @@ case "${fv3_res}" in export layout_y_gfs=32 export nthreads_fv3=4 export nthreads_fv3_gfs=4 + export nthreads_ufs=4 + export nthreads_ufs_gfs=4 export cdmbgwd="4.0,0.05,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="0.625,14.1,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=0.13e-3 # setting for UGWPv1 non-stationary GWD @@ -420,7 +434,7 @@ if [[ "${skip_ww3}" == "false" ]]; then ntasks_ww3=40 nthreads_ww3=1 ;; - "uglo_m1g16") + "uglo_m1g16") ntasks_ww3=1000 nthreads_ww3=1 ;; @@ -441,27 +455,30 @@ fi # Set the name of the UFS (previously nems) configure template to use # Default ufs.configure templates for supported model configurations +if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then + tmpl_suffix="_esmf" +fi case "${model_list}" in atm) - default_template="${PARMgfs}/ufs/ufs.configure.atm.IN" + default_template="${PARMgfs}/ufs/ufs.configure.atm${tmpl_suffix:-}.IN" ;; atm.aero) - default_template="${PARMgfs}/ufs/ufs.configure.atmaero.IN" + default_template="${PARMgfs}/ufs/ufs.configure.atmaero${tmpl_suffix:-}.IN" ;; atm.wave) - default_template="${PARMgfs}/ufs/ufs.configure.leapfrog_atm_wav.IN" + default_template="${PARMgfs}/ufs/ufs.configure.leapfrog_atm_wav${tmpl_suffix:-}.IN" ;; atm.ocean.ice) - default_template="${PARMgfs}/ufs/ufs.configure.s2s_esmf.IN" + default_template="${PARMgfs}/ufs/ufs.configure.s2s${tmpl_suffix:-}.IN" ;; atm.ocean.ice.aero) - default_template="${PARMgfs}/ufs/ufs.configure.s2sa_esmf.IN" + default_template="${PARMgfs}/ufs/ufs.configure.s2sa${tmpl_suffix:-}.IN" ;; atm.ocean.ice.wave) - default_template="${PARMgfs}/ufs/ufs.configure.s2sw_esmf.IN" + default_template="${PARMgfs}/ufs/ufs.configure.s2sw${tmpl_suffix:-}.IN" ;; atm.ocean.ice.wave.aero) - default_template="${PARMgfs}/ufs/ufs.configure.s2swa_esmf.IN" + default_template="${PARMgfs}/ufs/ufs.configure.s2swa${tmpl_suffix:-}.IN" ;; *) echo "FATAL ERROR: Unsupported UFSWM configuration for ${model_list}" diff --git a/parm/ufs/fv3/diag_table_aod b/parm/ufs/fv3/diag_table_aod index 0de51b66d8..fd8aee1791 100644 --- a/parm/ufs/fv3/diag_table_aod +++ b/parm/ufs/fv3/diag_table_aod @@ -3,4 +3,4 @@ "gfs_phys", "SU_AOD_550", "su_aod550", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "BC_AOD_550", "bc_aod550", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "OC_AOD_550", "oc_aod550", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "SS_AOD_550", "ss_aod550", "fv3_history2d", "all", .false., "none", 2 \ No newline at end of file +"gfs_phys", "SS_AOD_550", "ss_aod550", "fv3_history2d", "all", .false., "none", 2 diff --git a/scripts/exglobal_forecast.sh b/scripts/exglobal_forecast.sh index 0a6329910e..3555d4ef33 100755 --- a/scripts/exglobal_forecast.sh +++ b/scripts/exglobal_forecast.sh @@ -149,6 +149,12 @@ if [[ "${esmf_profile:-}" = ".true." ]]; then export ESMF_RUNTIME_PROFILE_OUTPUT=SUMMARY fi +if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then + unset OMP_NUM_THREADS +else + export OMP_NUM_THREADS=${UFS_THREADS:-1} +fi + ${NCP} "${EXECgfs}/${FCSTEXEC}" "${DATA}/" ${APRUN_UFS} "${DATA}/${FCSTEXEC}" 1>&1 2>&2 export ERR=$? diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 2b54f3ea10..9184e08432 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -166,12 +166,19 @@ declare -a ufs_templates=("model_configure.IN" \ "MOM6_data_table.IN" \ "ice_in.IN" \ "ufs.configure.atm.IN" \ + "ufs.configure.atm_esmf.IN" \ "ufs.configure.atmaero.IN" \ - "ufs.configure.leapfrog_atm_wav.IN" \ + "ufs.configure.atmaero_esmf.IN" \ + "ufs.configure.s2s.IN" \ "ufs.configure.s2s_esmf.IN" \ + "ufs.configure.s2sa.IN" \ "ufs.configure.s2sa_esmf.IN" \ + "ufs.configure.s2sw.IN" \ "ufs.configure.s2sw_esmf.IN" \ - "ufs.configure.s2swa_esmf.IN" ) + "ufs.configure.s2swa.IN" \ + "ufs.configure.s2swa_esmf.IN" \ + "ufs.configure.leapfrog_atm_wav.IN" \ + "ufs.configure.leapfrog_atm_wav_esmf.IN" ) for file in "${ufs_templates[@]}"; do [[ -s "${file}" ]] && rm -f "${file}" ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/tests/parm/${file}" . diff --git a/sorc/ufs_model.fd b/sorc/ufs_model.fd index 6988662728..126db7a514 160000 --- a/sorc/ufs_model.fd +++ b/sorc/ufs_model.fd @@ -1 +1 @@ -Subproject commit 698866272846e8c0f8f61ddb1b20d6463460cd63 +Subproject commit 126db7a5145319cb6a8b3d272215efcd5b88562f diff --git a/ush/module-setup.sh b/ush/module-setup.sh index 008ef7ee97..b66e3622d0 100755 --- a/ush/module-setup.sh +++ b/ush/module-setup.sh @@ -36,10 +36,10 @@ elif [[ ${MACHINE_ID} = orion* ]] ; then if ( ! eval module help > /dev/null 2>&1 ) ; then source /apps/lmod/lmod/init/bash fi - export LMOD_SYSTEM_DEFAULT_MODULES=contrib - set +u - module reset - set -u + #export LMOD_SYSTEM_DEFAULT_MODULES=git/2.28.0 # contrib has a lot of stuff we shouldn't put in MODULEPATH + #set +u + module purge # reset causes issues on Orion sometimes. + #set -u elif [[ ${MACHINE_ID} = s4* ]] ; then # We are on SSEC Wisconsin S4