Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add option to use traditional threading in the UFS #2384

Merged
Merged
Show file tree
Hide file tree
Changes from 15 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions parm/config/gefs/config.fcst
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@

echo "BEGIN: config.fcst"

export USE_ESMF_THREADING="YES" # Toggle to use ESMF-managed threading or traditional threading in UFSWM
WalterKolczynski-NOAA marked this conversation as resolved.
Show resolved Hide resolved

# Turn off waves if not used for this CDUMP
case ${WAVE_CDUMP} in
both | "${CDUMP/enkf}" ) ;; # Don't change
Expand Down
18 changes: 16 additions & 2 deletions parm/config/gefs/config.resources
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,20 @@ case ${step} in
ntasks_fv3=${ntasks_fv3_gfs}
ntasks_quilt=${ntasks_quilt_gfs}
nthreads_fv3=${nthreads_fv3_gfs}
nthreads_ufs=${nthreads_ufs_gfs}
fi

# Determine if using ESMF-managed threading or traditional threading
# If using traditional threading, set them to 1
if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then
export UFS_THREADS=1
else # traditional threading
export UFS_THREADS=${nthreads_ufs:-1}
nthreads_fv3=1
nthreads_mediator=1
[[ "${DO_WAVE}" == "YES" ]] && nthreads_ww3=1
[[ "${DO_OCN}" == "YES" ]] && nthreads_mom6=1
[[ "${DO_ICE}" == "YES" ]] && nthreads_cice6=1
fi

# PETS for the atmosphere dycore
Expand Down Expand Up @@ -177,11 +191,11 @@ case ${step} in

if [[ "${_CDUMP}" =~ "gfs" ]]; then
declare -x "npe_${step}_gfs"="${NTASKS_TOT}"
declare -x "nth_${step}_gfs"=1 # ESMF handles threading for the UFS-weather-model
declare -x "nth_${step}_gfs"="${UFS_THREADS}"
declare -x "npe_node_${step}_gfs"="${npe_node_max}"
else
declare -x "npe_${step}"="${NTASKS_TOT}"
declare -x "nth_${step}"=1 # ESMF handles threading for the UFS-weather-model
declare -x "nth_${step}"="${UFS_THREADS}"
declare -x "npe_node_${step}"="${npe_node_max}"
fi

Expand Down
31 changes: 24 additions & 7 deletions parm/config/gefs/config.ufs
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,8 @@ case "${fv3_res}" in
export layout_y_gfs=1
export nthreads_fv3=1
export nthreads_fv3_gfs=1
export nthreads_ufs=1
export nthreads_ufs_gfs=1
export cdmbgwd="0.071,2.1,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling
export cdmbgwd_gsl="40.0,1.77,1.0,1.0" # settings for GSL drag suite
export knob_ugwp_tauamp=6.0e-3 # setting for UGWPv1 non-stationary GWD
Expand All @@ -94,6 +96,8 @@ case "${fv3_res}" in
export layout_y_gfs=2
export nthreads_fv3=1
export nthreads_fv3_gfs=1
export nthreads_ufs=1
export nthreads_ufs_gfs=1
export cdmbgwd="0.14,1.8,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling
export cdmbgwd_gsl="20.0,2.5,1.0,1.0" # settings for GSL drag suite
export knob_ugwp_tauamp=3.0e-3 # setting for UGWPv1 non-stationary GWD
Expand All @@ -110,6 +114,8 @@ case "${fv3_res}" in
export layout_y_gfs=6
export nthreads_fv3=1
export nthreads_fv3_gfs=2
export nthreads_ufs=1
export nthreads_ufs_gfs=2
export cdmbgwd="0.23,1.5,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling
export cdmbgwd_gsl="10.0,3.5,1.0,1.0" # settings for GSL drag suite
export knob_ugwp_tauamp=1.5e-3 # setting for UGWPv1 non-stationary GWD
Expand All @@ -126,6 +132,8 @@ case "${fv3_res}" in
export layout_y_gfs=8
export nthreads_fv3=1
export nthreads_fv3_gfs=2
export nthreads_ufs=1
export nthreads_ufs_gfs=2
export cdmbgwd="1.1,0.72,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling
export cdmbgwd_gsl="5.0,5.0,1.0,1.0" # settings for GSL drag suite
export knob_ugwp_tauamp=0.8e-3 # setting for UGWPv1 non-stationary GWD
Expand All @@ -142,6 +150,8 @@ case "${fv3_res}" in
export layout_y_gfs=16
export nthreads_fv3=4
export nthreads_fv3_gfs=4
export nthreads_ufs=4
export nthreads_ufs_gfs=4
export cdmbgwd="4.0,0.15,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling
export cdmbgwd_gsl="2.5,7.5,1.0,1.0" # settings for GSL drag suite
export knob_ugwp_tauamp=0.5e-3 # setting for UGWPv1 non-stationary GWD
Expand All @@ -158,6 +168,8 @@ case "${fv3_res}" in
export layout_y_gfs=16
export nthreads_fv3=4
export nthreads_fv3_gfs=4
export nthreads_ufs=4
export nthreads_ufs_gfs=4
export cdmbgwd="4.0,0.10,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling
export cdmbgwd_gsl="1.67,8.8,1.0,1.0" # settings for GSL drag suite
export knob_ugwp_tauamp=0.35e-3 # setting for UGWPv1 non-stationary GWD
Expand All @@ -174,6 +186,8 @@ case "${fv3_res}" in
export layout_y_gfs=32
export nthreads_fv3=4
export nthreads_fv3_gfs=4
export nthreads_ufs=4
export nthreads_ufs_gfs=4
export cdmbgwd="4.0,0.05,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling
export cdmbgwd_gsl="0.625,14.1,1.0,1.0" # settings for GSL drag suite
export knob_ugwp_tauamp=0.13e-3 # setting for UGWPv1 non-stationary GWD
Expand Down Expand Up @@ -433,27 +447,30 @@ fi

# Set the name of the UFS (previously nems) configure template to use
# Default ufs.configure templates for supported model configurations
if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then
tmpl_suffix="_esmf"
fi
case "${model_list}" in
atm)
default_template="${PARMgfs}/ufs/ufs.configure.atm.IN"
default_template="${PARMgfs}/ufs/ufs.configure.atm${tmpl_suffix:-}.IN"
;;
atm.aero)
default_template="${PARMgfs}/ufs/ufs.configure.atmaero.IN"
default_template="${PARMgfs}/ufs/ufs.configure.atmaero${tmpl_suffix:-}.IN"
;;
atm.wave)
default_template="${PARMgfs}/ufs/ufs.configure.leapfrog_atm_wav.IN"
default_template="${PARMgfs}/ufs/ufs.configure.leapfrog_atm_wav${tmpl_suffix:-}.IN"
;;
atm.ocean.ice)
default_template="${PARMgfs}/ufs/ufs.configure.s2s_esmf.IN"
default_template="${PARMgfs}/ufs/ufs.configure.s2s${tmpl_suffix:-}.IN"
;;
atm.ocean.ice.aero)
default_template="${PARMgfs}/ufs/ufs.configure.s2sa_esmf.IN"
default_template="${PARMgfs}/ufs/ufs.configure.s2sa${tmpl_suffix:-}.IN"
;;
atm.ocean.ice.wave)
default_template="${PARMgfs}/ufs/ufs.configure.s2sw_esmf.IN"
default_template="${PARMgfs}/ufs/ufs.configure.s2sw${tmpl_suffix:-}.IN"
;;
atm.ocean.ice.wave.aero)
default_template="${PARMgfs}/ufs/ufs.configure.s2swa_esmf.IN"
default_template="${PARMgfs}/ufs/ufs.configure.s2swa${tmpl_suffix:-}.IN"
;;
*)
echo "FATAL ERROR: Unsupported UFSWM configuration for ${model_list}"
Expand Down
2 changes: 2 additions & 0 deletions parm/config/gfs/config.fcst
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@

echo "BEGIN: config.fcst"

export USE_ESMF_THREADING="YES" # Toggle to use ESMF-managed threading or traditional threading in UFSWM

# Turn off waves if not used for this CDUMP
case ${WAVE_CDUMP} in
both | "${CDUMP/enkf}" ) ;; # Don't change
Expand Down
26 changes: 22 additions & 4 deletions parm/config/gfs/config.resources
Original file line number Diff line number Diff line change
Expand Up @@ -570,6 +570,20 @@ case ${step} in
ntasks_fv3=${ntasks_fv3_gfs}
ntasks_quilt=${ntasks_quilt_gfs}
nthreads_fv3=${nthreads_fv3_gfs}
nthreads_ufs=${nthreads_ufs_gfs}
fi

# Determine if using ESMF-managed threading or traditional threading
# If using traditional threading, set them to 1
if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then
export UFS_THREADS=1
else # traditional threading
export UFS_THREADS=${nthreads_ufs:-1}
nthreads_fv3=1
nthreads_mediator=1
[[ "${DO_WAVE}" == "YES" ]] && nthreads_ww3=1
[[ "${DO_OCN}" == "YES" ]] && nthreads_mom6=1
[[ "${DO_ICE}" == "YES" ]] && nthreads_cice6=1
fi

# PETS for the atmosphere dycore
Expand Down Expand Up @@ -646,11 +660,11 @@ case ${step} in

if [[ "${_CDUMP}" =~ "gfs" ]]; then
declare -x "npe_${step}_gfs"="${NTASKS_TOT}"
declare -x "nth_${step}_gfs"=1 # ESMF handles threading for the UFS-weather-model
declare -x "nth_${step}_gfs"="${UFS_THREADS}"
declare -x "npe_node_${step}_gfs"="${npe_node_max}"
else
declare -x "npe_${step}"="${NTASKS_TOT}"
declare -x "nth_${step}"=1 # ESMF handles threading for the UFS-weather-model
declare -x "nth_${step}"="${UFS_THREADS}"
declare -x "npe_node_${step}"="${npe_node_max}"
fi

Expand Down Expand Up @@ -692,9 +706,13 @@ case ${step} in
"C48" | "C96")
export npe_upp=${CASE:1}
;;
"C192" | "C384" | "C768")
"C192" | "C384")
export npe_upp=120
export memory_upp="96GB"
;;
"C768")
export npe_upp=120
export memory_upp="48GB"
if [[ ${machine} == "WCOSS2" ]]; then export memory_upp="480GB" ; fi
KateFriedman-NOAA marked this conversation as resolved.
Show resolved Hide resolved
aerorahul marked this conversation as resolved.
Show resolved Hide resolved
;;
*)
echo "FATAL ERROR: Resources not defined for job ${job} at resolution ${CASE}"
Expand Down
33 changes: 25 additions & 8 deletions parm/config/gfs/config.ufs
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,8 @@ case "${fv3_res}" in
export layout_y_gfs=1
export nthreads_fv3=1
export nthreads_fv3_gfs=1
export nthreads_ufs=1
export nthreads_ufs_gfs=1
export cdmbgwd="0.071,2.1,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling
export cdmbgwd_gsl="40.0,1.77,1.0,1.0" # settings for GSL drag suite
export knob_ugwp_tauamp=6.0e-3 # setting for UGWPv1 non-stationary GWD
Expand All @@ -94,6 +96,8 @@ case "${fv3_res}" in
export layout_y_gfs=2
export nthreads_fv3=1
export nthreads_fv3_gfs=1
export nthreads_ufs=1
export nthreads_ufs_gfs=1
export cdmbgwd="0.14,1.8,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling
export cdmbgwd_gsl="20.0,2.5,1.0,1.0" # settings for GSL drag suite
export knob_ugwp_tauamp=3.0e-3 # setting for UGWPv1 non-stationary GWD
Expand All @@ -110,6 +114,8 @@ case "${fv3_res}" in
export layout_y_gfs=6
export nthreads_fv3=1
export nthreads_fv3_gfs=2
export nthreads_ufs=1
export nthreads_ufs_gfs=2
export cdmbgwd="0.23,1.5,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling
export cdmbgwd_gsl="10.0,3.5,1.0,1.0" # settings for GSL drag suite
export knob_ugwp_tauamp=1.5e-3 # setting for UGWPv1 non-stationary GWD
Expand All @@ -126,6 +132,8 @@ case "${fv3_res}" in
export layout_y_gfs=8
export nthreads_fv3=2
export nthreads_fv3_gfs=2
export nthreads_ufs=2
export nthreads_ufs_gfs=2
export cdmbgwd="1.1,0.72,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling
export cdmbgwd_gsl="5.0,5.0,1.0,1.0" # settings for GSL drag suite
export knob_ugwp_tauamp=0.8e-3 # setting for UGWPv1 non-stationary GWD
Expand All @@ -142,6 +150,8 @@ case "${fv3_res}" in
export layout_y_gfs=16
export nthreads_fv3=4
export nthreads_fv3_gfs=4
export nthreads_ufs=4
export nthreads_ufs_gfs=4
export cdmbgwd="4.0,0.15,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling
export cdmbgwd_gsl="2.5,7.5,1.0,1.0" # settings for GSL drag suite
export knob_ugwp_tauamp=0.5e-3 # setting for UGWPv1 non-stationary GWD
Expand All @@ -158,6 +168,8 @@ case "${fv3_res}" in
export layout_y_gfs=16
export nthreads_fv3=4
export nthreads_fv3_gfs=4
export nthreads_ufs=4
export nthreads_ufs_gfs=4
export cdmbgwd="4.0,0.10,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling
export cdmbgwd_gsl="1.67,8.8,1.0,1.0" # settings for GSL drag suite
export knob_ugwp_tauamp=0.35e-3 # setting for UGWPv1 non-stationary GWD
Expand All @@ -174,6 +186,8 @@ case "${fv3_res}" in
export layout_y_gfs=32
export nthreads_fv3=4
export nthreads_fv3_gfs=4
export nthreads_ufs=4
export nthreads_ufs_gfs=4
export cdmbgwd="4.0,0.05,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling
export cdmbgwd_gsl="0.625,14.1,1.0,1.0" # settings for GSL drag suite
export knob_ugwp_tauamp=0.13e-3 # setting for UGWPv1 non-stationary GWD
Expand Down Expand Up @@ -420,7 +434,7 @@ if [[ "${skip_ww3}" == "false" ]]; then
ntasks_ww3=40
nthreads_ww3=1
;;
"uglo_m1g16")
"uglo_m1g16")
ntasks_ww3=1000
nthreads_ww3=1
;;
Expand All @@ -441,27 +455,30 @@ fi

# Set the name of the UFS (previously nems) configure template to use
# Default ufs.configure templates for supported model configurations
if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then
tmpl_suffix="_esmf"
fi
case "${model_list}" in
atm)
default_template="${PARMgfs}/ufs/ufs.configure.atm.IN"
default_template="${PARMgfs}/ufs/ufs.configure.atm${tmpl_suffix:-}.IN"
;;
atm.aero)
default_template="${PARMgfs}/ufs/ufs.configure.atmaero.IN"
default_template="${PARMgfs}/ufs/ufs.configure.atmaero${tmpl_suffix:-}.IN"
;;
atm.wave)
default_template="${PARMgfs}/ufs/ufs.configure.leapfrog_atm_wav.IN"
default_template="${PARMgfs}/ufs/ufs.configure.leapfrog_atm_wav${tmpl_suffix:-}.IN"
;;
atm.ocean.ice)
default_template="${PARMgfs}/ufs/ufs.configure.s2s_esmf.IN"
default_template="${PARMgfs}/ufs/ufs.configure.s2s${tmpl_suffix:-}.IN"
;;
atm.ocean.ice.aero)
default_template="${PARMgfs}/ufs/ufs.configure.s2sa_esmf.IN"
default_template="${PARMgfs}/ufs/ufs.configure.s2sa${tmpl_suffix:-}.IN"
;;
atm.ocean.ice.wave)
default_template="${PARMgfs}/ufs/ufs.configure.s2sw_esmf.IN"
default_template="${PARMgfs}/ufs/ufs.configure.s2sw${tmpl_suffix:-}.IN"
;;
atm.ocean.ice.wave.aero)
default_template="${PARMgfs}/ufs/ufs.configure.s2swa_esmf.IN"
default_template="${PARMgfs}/ufs/ufs.configure.s2swa${tmpl_suffix:-}.IN"
;;
*)
echo "FATAL ERROR: Unsupported UFSWM configuration for ${model_list}"
Expand Down
2 changes: 1 addition & 1 deletion parm/ufs/fv3/diag_table_aod
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,4 @@
"gfs_phys", "SU_AOD_550", "su_aod550", "fv3_history2d", "all", .false., "none", 2
"gfs_phys", "BC_AOD_550", "bc_aod550", "fv3_history2d", "all", .false., "none", 2
"gfs_phys", "OC_AOD_550", "oc_aod550", "fv3_history2d", "all", .false., "none", 2
"gfs_phys", "SS_AOD_550", "ss_aod550", "fv3_history2d", "all", .false., "none", 2
"gfs_phys", "SS_AOD_550", "ss_aod550", "fv3_history2d", "all", .false., "none", 2
6 changes: 6 additions & 0 deletions scripts/exglobal_forecast.sh
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,12 @@ if [[ "${esmf_profile:-}" = ".true." ]]; then
export ESMF_RUNTIME_PROFILE_OUTPUT=SUMMARY
fi

if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then
unset OMP_NUM_THREADS
else
export OMP_NUM_THREADS=${UFS_THREADS:-1}
fi

${NCP} "${EXECgfs}/${FCSTEXEC}" "${DATA}/"
${APRUN_UFS} "${DATA}/${FCSTEXEC}" 1>&1 2>&2
export ERR=$?
Expand Down
11 changes: 9 additions & 2 deletions sorc/link_workflow.sh
Original file line number Diff line number Diff line change
Expand Up @@ -166,12 +166,19 @@ declare -a ufs_templates=("model_configure.IN" \
"MOM6_data_table.IN" \
"ice_in.IN" \
"ufs.configure.atm.IN" \
"ufs.configure.atm_esmf.IN" \
"ufs.configure.atmaero.IN" \
"ufs.configure.leapfrog_atm_wav.IN" \
"ufs.configure.atmaero_esmf.IN" \
"ufs.configure.s2s.IN" \
"ufs.configure.s2s_esmf.IN" \
"ufs.configure.s2sa.IN" \
"ufs.configure.s2sa_esmf.IN" \
"ufs.configure.s2sw.IN" \
"ufs.configure.s2sw_esmf.IN" \
"ufs.configure.s2swa_esmf.IN" )
"ufs.configure.s2swa.IN" \
"ufs.configure.s2swa_esmf.IN" \
"ufs.configure.leapfrog_atm_wav.IN" \
"ufs.configure.leapfrog_atm_wav_esmf.IN" )
for file in "${ufs_templates[@]}"; do
[[ -s "${file}" ]] && rm -f "${file}"
${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/tests/parm/${file}" .
Expand Down
2 changes: 1 addition & 1 deletion sorc/ufs_model.fd
Submodule ufs_model.fd updated 50 files
+1 −1 CICE-interface/CICE
+1 −1 CMEPS-interface/CMEPS
+1 −1 FV3
+20 −7 LICENSE.md
+1 −1 MOM6-interface/MOM6
+6 −1 MOM6-interface/mom6_files.cmake
+9 −6 README.md
+11 −0 cmake/GNU.cmake
+7 −3 doc/UsersGuide/Makefile
+3 −0 doc/UsersGuide/requirements.in
+82 −3 doc/UsersGuide/requirements.txt
+1 −1 doc/UsersGuide/source/CodeOverview.rst
+1 −1 doc/UsersGuide/source/Configurations.rst
+2 −2 doc/UsersGuide/source/Glossary.rst
+1 −1 doc/UsersGuide/source/InputsOutputs.rst
+13 −6 doc/UsersGuide/source/conf.py
+1 −1 doc/UsersGuide/source/references.bib
+2 −2 modulefiles/ufs_gaea.intel.lua
+1 −1 modulefiles/ufs_jet.intel.lua
+1 −1 tests/bl_date.conf
+301 −0 tests/ci/Jenkinsfile.combined
+1 −1 tests/default_vars.sh
+1 −1 tests/fv3_conf/fv3_slurm.IN_gaea
+24 −24 tests/logs/OpnReqTests_control_p8_hera.log
+12 −12 tests/logs/OpnReqTests_cpld_control_nowave_noaero_p8_hera.log
+12 −12 tests/logs/OpnReqTests_regional_control_hera.log
+2,467 −0 tests/logs/RT-run-Orion.log
+287 −255 tests/logs/RegressionTests_derecho.log
+292 −269 tests/logs/RegressionTests_gaea.log
+360 −360 tests/logs/RegressionTests_hera.log
+363 −340 tests/logs/RegressionTests_hercules.log
+254 −285 tests/logs/RegressionTests_jet.log
+374 −269 tests/logs/RegressionTests_orion.log
+251 −228 tests/logs/RegressionTests_wcoss2.log
+2 −2 tests/parm/MOM_input_025.IN
+2 −2 tests/parm/MOM_input_050.IN
+2 −2 tests/parm/MOM_input_100.IN
+2 −2 tests/parm/MOM_input_hafs
+1 −2 tests/parm/ufs.configure.atm.IN
+27 −0 tests/parm/ufs.configure.atm_esmf.IN
+1 −3 tests/parm/ufs.configure.atmaero.IN
+40 −0 tests/parm/ufs.configure.atmaero_esmf.IN
+1 −3 tests/parm/ufs.configure.leapfrog_atm_wav.IN
+41 −0 tests/parm/ufs.configure.leapfrog_atm_wav_esmf.IN
+7 −7 tests/rt.conf
+73 −34 tests/rt.sh
+1 −1 tests/tests/atmaero_control_p8
+1 −1 tests/tests/atmaero_control_p8_rad
+1 −1 tests/tests/atmaero_control_p8_rad_micro
+1 −1 tests/tests/control_c384gdas_wav
Loading