From 102df72bc8f99e51f00d1755db33a923e462d3ed Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Tue, 6 Feb 2024 12:02:26 -0500 Subject: [PATCH 01/14] add toggle for ESMF/traditional threading for UFS --- parm/config/gefs/config.fcst | 1 + parm/config/gefs/config.resources | 16 +++++++++++++++- parm/config/gefs/config.ufs | 14 ++++++++++++++ parm/config/gfs/config.fcst | 1 + parm/config/gfs/config.resources | 15 ++++++++++++++- parm/config/gfs/config.ufs | 14 ++++++++++++++ scripts/exglobal_forecast.sh | 6 ++++++ 7 files changed, 65 insertions(+), 2 deletions(-) diff --git a/parm/config/gefs/config.fcst b/parm/config/gefs/config.fcst index 6a2a852e0b..224e961c16 100644 --- a/parm/config/gefs/config.fcst +++ b/parm/config/gefs/config.fcst @@ -37,6 +37,7 @@ if [[ "${DONST}" == "YES" ]]; then source "${EXPDIR}/config.nsst" fi +export USE_ESMF_THREADING="YES" # Toggle to use ESMF-managed threading or traditional threading in UFSWM export esmf_profile=".false." export esmf_logkind="ESMF_LOGKIND_MULTI_ON_ERROR" #Options: ESMF_LOGKIND_MULTI_ON_ERROR, ESMF_LOGKIND_MULTI, ESMF_LOGKIND_NONE diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources index 36b70aecb8..e300252d39 100644 --- a/parm/config/gefs/config.resources +++ b/parm/config/gefs/config.resources @@ -107,6 +107,20 @@ case ${step} in ntasks_fv3=${ntasks_fv3_gfs} ntasks_quilt=${ntasks_quilt_gfs} nthreads_fv3=${nthreads_fv3_gfs} + nthreads_ufs=${nthreads_ufs_gfs} + fi + + # Determine if using ESMF-managed threading or traditional threading + # If using traditional threading, set them to 1 + if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then + export UFS_THREADS=1 + else # traditional threading + export UFS_THREADS=${nthreads_ufs:-1} + nthreads_fv3=1 + nthreads_mediator=1 + [[ "${DO_WAVE}" == "YES" ]] && nthreads_ww3=1 + [[ "${DO_OCN}" == "YES" ]] && nthreads_mom6=1 + [[ "${DO_ICE}" == "YES" ]] && nthreads_cice6=1 fi # PETS for the atmosphere dycore @@ -239,4 +253,4 @@ case ${step} in esac -echo "END: config.resources" \ No newline at end of file +echo "END: config.resources" diff --git a/parm/config/gefs/config.ufs b/parm/config/gefs/config.ufs index 2031d0b538..76c2060f8d 100644 --- a/parm/config/gefs/config.ufs +++ b/parm/config/gefs/config.ufs @@ -126,6 +126,8 @@ case "${fv3_res}" in export layout_y_gfs=1 export nthreads_fv3=1 export nthreads_fv3_gfs=1 + export nthreads_ufs=1 + export nthreads_ufs_gfs=1 export cdmbgwd="0.071,2.1,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="40.0,1.77,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=6.0e-3 # setting for UGWPv1 non-stationary GWD @@ -142,6 +144,8 @@ case "${fv3_res}" in export layout_y_gfs=2 export nthreads_fv3=1 export nthreads_fv3_gfs=1 + export nthreads_ufs=1 + export nthreads_ufs_gfs=1 export cdmbgwd="0.14,1.8,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="20.0,2.5,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=3.0e-3 # setting for UGWPv1 non-stationary GWD @@ -158,6 +162,8 @@ case "${fv3_res}" in export layout_y_gfs=6 export nthreads_fv3=1 export nthreads_fv3_gfs=2 + export nthreads_ufs=1 + export nthreads_ufs_gfs=2 export cdmbgwd="0.23,1.5,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="10.0,3.5,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=1.5e-3 # setting for UGWPv1 non-stationary GWD @@ -174,6 +180,8 @@ case "${fv3_res}" in export layout_y_gfs=8 export nthreads_fv3=1 export nthreads_fv3_gfs=2 + export nthreads_ufs=1 + export nthreads_ufs_gfs=2 export cdmbgwd="1.1,0.72,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="5.0,5.0,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=0.8e-3 # setting for UGWPv1 non-stationary GWD @@ -190,6 +198,8 @@ case "${fv3_res}" in export layout_y_gfs=16 export nthreads_fv3=4 export nthreads_fv3_gfs=4 + export nthreads_ufs=4 + export nthreads_ufs_gfs=4 export cdmbgwd="4.0,0.15,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="2.5,7.5,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=0.5e-3 # setting for UGWPv1 non-stationary GWD @@ -206,6 +216,8 @@ case "${fv3_res}" in export layout_y_gfs=16 export nthreads_fv3=4 export nthreads_fv3_gfs=4 + export nthreads_ufs=4 + export nthreads_ufs_gfs=4 export cdmbgwd="4.0,0.10,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="1.67,8.8,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=0.35e-3 # setting for UGWPv1 non-stationary GWD @@ -222,6 +234,8 @@ case "${fv3_res}" in export layout_y_gfs=32 export nthreads_fv3=4 export nthreads_fv3_gfs=4 + export nthreads_ufs=4 + export nthreads_ufs_gfs=4 export cdmbgwd="4.0,0.05,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="0.625,14.1,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=0.13e-3 # setting for UGWPv1 non-stationary GWD diff --git a/parm/config/gfs/config.fcst b/parm/config/gfs/config.fcst index d2e2664e9c..4a6f932ac9 100644 --- a/parm/config/gfs/config.fcst +++ b/parm/config/gfs/config.fcst @@ -49,6 +49,7 @@ if [[ "${DONST}" == "YES" ]]; then source "${EXPDIR}/config.nsst" fi +export USE_ESMF_THREADING="YES" # Toggle to use ESMF-managed threading or traditional threading in UFSWM export esmf_profile=".false." export esmf_logkind="ESMF_LOGKIND_MULTI_ON_ERROR" #Options: ESMF_LOGKIND_MULTI_ON_ERROR, ESMF_LOGKIND_MULTI, ESMF_LOGKIND_NONE diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 1dcf757de2..69df8b724e 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -537,6 +537,19 @@ case ${step} in nthreads_fv3=${nthreads_fv3_gfs} fi + # Determine if using ESMF-managed threading or traditional threading + # If using traditional threading, set them to 1 + if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then + export UFS_THREADS=1 + else # traditional threading + export UFS_THREADS=${nthreads_ufs:-1} + nthreads_fv3=1 + nthreads_mediator=1 + [[ "${DO_WAVE}" == "YES" ]] && nthreads_ww3=1 + [[ "${DO_OCN}" == "YES" ]] && nthreads_mom6=1 + [[ "${DO_ICE}" == "YES" ]] && nthreads_cice6=1 + fi + # PETS for the atmosphere dycore (( FV3PETS = ntasks_fv3 * nthreads_fv3 )) echo "FV3 using (nthreads, PETS) = (${nthreads_fv3}, ${FV3PETS})" @@ -926,7 +939,7 @@ case ${step} in export nth_eupd=4 fi ;; - *) + *) echo "FATAL ERROR: Resources not defined for job ${job} at resolution ${CASE}" exit 4 ;; diff --git a/parm/config/gfs/config.ufs b/parm/config/gfs/config.ufs index 0a59da47ca..0a8f2f9d15 100644 --- a/parm/config/gfs/config.ufs +++ b/parm/config/gfs/config.ufs @@ -126,6 +126,8 @@ case "${fv3_res}" in export layout_y_gfs=1 export nthreads_fv3=1 export nthreads_fv3_gfs=1 + export nthreads_ufs=1 + export nthreads_ufs_gfs=1 export cdmbgwd="0.071,2.1,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="40.0,1.77,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=6.0e-3 # setting for UGWPv1 non-stationary GWD @@ -142,6 +144,8 @@ case "${fv3_res}" in export layout_y_gfs=2 export nthreads_fv3=1 export nthreads_fv3_gfs=1 + export nthreads_ufs=1 + export nthreads_ufs_gfs=1 export cdmbgwd="0.14,1.8,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="20.0,2.5,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=3.0e-3 # setting for UGWPv1 non-stationary GWD @@ -158,6 +162,8 @@ case "${fv3_res}" in export layout_y_gfs=6 export nthreads_fv3=1 export nthreads_fv3_gfs=2 + export nthreads_ufs=1 + export nthreads_ufs_gfs=2 export cdmbgwd="0.23,1.5,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="10.0,3.5,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=1.5e-3 # setting for UGWPv1 non-stationary GWD @@ -174,6 +180,8 @@ case "${fv3_res}" in export layout_y_gfs=8 export nthreads_fv3=2 export nthreads_fv3_gfs=2 + export nthreads_ufs=2 + export nthreads_ufs_gfs=2 export cdmbgwd="1.1,0.72,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="5.0,5.0,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=0.8e-3 # setting for UGWPv1 non-stationary GWD @@ -190,6 +198,8 @@ case "${fv3_res}" in export layout_y_gfs=16 export nthreads_fv3=4 export nthreads_fv3_gfs=4 + export nthreads_ufs=4 + export nthreads_ufs_gfs=4 export cdmbgwd="4.0,0.15,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="2.5,7.5,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=0.5e-3 # setting for UGWPv1 non-stationary GWD @@ -206,6 +216,8 @@ case "${fv3_res}" in export layout_y_gfs=16 export nthreads_fv3=4 export nthreads_fv3_gfs=4 + export nthreads_ufs=4 + export nthreads_ufs_gfs=4 export cdmbgwd="4.0,0.10,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="1.67,8.8,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=0.35e-3 # setting for UGWPv1 non-stationary GWD @@ -222,6 +234,8 @@ case "${fv3_res}" in export layout_y_gfs=32 export nthreads_fv3=4 export nthreads_fv3_gfs=4 + export nthreads_ufs=4 + export nthreads_ufs_gfs=4 export cdmbgwd="4.0,0.05,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="0.625,14.1,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=0.13e-3 # setting for UGWPv1 non-stationary GWD diff --git a/scripts/exglobal_forecast.sh b/scripts/exglobal_forecast.sh index eebc9e59c3..276d4c8a7c 100755 --- a/scripts/exglobal_forecast.sh +++ b/scripts/exglobal_forecast.sh @@ -146,6 +146,12 @@ if [[ "${esmf_profile:-}" = ".true." ]]; then export ESMF_RUNTIME_PROFILE_OUTPUT=SUMMARY fi +if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then + unset OMP_NUM_THREADS +else + export OMP_NUM_THREADS=${UFS_THREADS:-1} +fi + ${NCP} "${FCSTEXECDIR}/${FCSTEXEC}" "${DATA}/" ${APRUN_UFS} "${DATA}/${FCSTEXEC}" 1>&1 2>&2 export ERR=$? From 1d71b231a48070f1dbcf7188b7309ef02b6a8786 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Wed, 7 Feb 2024 12:55:55 -0500 Subject: [PATCH 02/14] choose the appropriate template based on threading choice for ufswm --- parm/config/gefs/config.fcst | 3 ++- parm/config/gefs/config.ufs | 17 ++++++++++------- parm/config/gfs/config.fcst | 3 ++- parm/config/gfs/config.ufs | 17 ++++++++++------- sorc/link_workflow.sh | 14 ++++++++++---- 5 files changed, 34 insertions(+), 20 deletions(-) diff --git a/parm/config/gefs/config.fcst b/parm/config/gefs/config.fcst index 224e961c16..9f00739957 100644 --- a/parm/config/gefs/config.fcst +++ b/parm/config/gefs/config.fcst @@ -5,6 +5,8 @@ echo "BEGIN: config.fcst" +export USE_ESMF_THREADING="YES" # Toggle to use ESMF-managed threading or traditional threading in UFSWM + # Turn off waves if not used for this CDUMP case ${WAVE_CDUMP} in both | "${CDUMP/enkf}" ) ;; # Don't change @@ -37,7 +39,6 @@ if [[ "${DONST}" == "YES" ]]; then source "${EXPDIR}/config.nsst" fi -export USE_ESMF_THREADING="YES" # Toggle to use ESMF-managed threading or traditional threading in UFSWM export esmf_profile=".false." export esmf_logkind="ESMF_LOGKIND_MULTI_ON_ERROR" #Options: ESMF_LOGKIND_MULTI_ON_ERROR, ESMF_LOGKIND_MULTI, ESMF_LOGKIND_NONE diff --git a/parm/config/gefs/config.ufs b/parm/config/gefs/config.ufs index 76c2060f8d..deedd2525e 100644 --- a/parm/config/gefs/config.ufs +++ b/parm/config/gefs/config.ufs @@ -495,27 +495,30 @@ fi # Set the name of the UFS (previously nems) configure template to use # Default ufs.configure templates for supported model configurations +if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then + tmpl_suffix="_esmf" +fi case "${model_list}" in atm) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.atm.IN" + default_template="${HOMEgfs}/parm/ufs/ufs.configure.atm${tmpl_suffix:-}.IN" ;; atm.aero) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.atmaero.IN" + default_template="${HOMEgfs}/parm/ufs/ufs.configure.atmaero${tmpl_suffix:-}.IN" ;; atm.wave) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.leapfrog_atm_wav.IN" + default_template="${HOMEgfs}/parm/ufs/ufs.configure.leapfrog_atm_wav${tmpl_suffix:-}.IN" ;; atm.ocean.ice) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2s_esmf.IN" + default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2s${tmpl_suffix:-}.IN" ;; atm.ocean.ice.aero) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2sa_esmf.IN" + default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2sa${tmpl_suffix:-}.IN" ;; atm.ocean.ice.wave) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2sw_esmf.IN" + default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2sw${tmpl_suffix:-}.IN" ;; atm.ocean.ice.wave.aero) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2swa_esmf.IN" + default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2swa${tmpl_suffix:-}.IN" ;; *) echo "FATAL ERROR: Unsupported UFSWM configuration for ${model_list}" diff --git a/parm/config/gfs/config.fcst b/parm/config/gfs/config.fcst index 4a6f932ac9..fbcb4029b6 100644 --- a/parm/config/gfs/config.fcst +++ b/parm/config/gfs/config.fcst @@ -5,6 +5,8 @@ echo "BEGIN: config.fcst" +export USE_ESMF_THREADING="YES" # Toggle to use ESMF-managed threading or traditional threading in UFSWM + # Turn off waves if not used for this CDUMP case ${WAVE_CDUMP} in both | "${CDUMP/enkf}" ) ;; # Don't change @@ -49,7 +51,6 @@ if [[ "${DONST}" == "YES" ]]; then source "${EXPDIR}/config.nsst" fi -export USE_ESMF_THREADING="YES" # Toggle to use ESMF-managed threading or traditional threading in UFSWM export esmf_profile=".false." export esmf_logkind="ESMF_LOGKIND_MULTI_ON_ERROR" #Options: ESMF_LOGKIND_MULTI_ON_ERROR, ESMF_LOGKIND_MULTI, ESMF_LOGKIND_NONE diff --git a/parm/config/gfs/config.ufs b/parm/config/gfs/config.ufs index 0a8f2f9d15..eff25ea3a7 100644 --- a/parm/config/gfs/config.ufs +++ b/parm/config/gfs/config.ufs @@ -495,27 +495,30 @@ fi # Set the name of the UFS (previously nems) configure template to use # Default ufs.configure templates for supported model configurations +if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then + tmpl_suffix="_esmf" +fi case "${model_list}" in atm) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.atm.IN" + default_template="${HOMEgfs}/parm/ufs/ufs.configure.atm${tmpl_suffix:-}.IN" ;; atm.aero) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.atmaero.IN" + default_template="${HOMEgfs}/parm/ufs/ufs.configure.atmaero${tmpl_suffix:-}.IN" ;; atm.wave) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.leapfrog_atm_wav.IN" + default_template="${HOMEgfs}/parm/ufs/ufs.configure.leapfrog_atm_wav${tmpl_suffix:-}.IN" ;; atm.ocean.ice) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2s_esmf.IN" + default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2s${tmpl_suffix:-}.IN" ;; atm.ocean.ice.aero) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2sa_esmf.IN" + default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2sa${tmpl_suffix:-}.IN" ;; atm.ocean.ice.wave) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2sw_esmf.IN" + default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2sw${tmpl_suffix:-}.IN" ;; atm.ocean.ice.wave.aero) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2swa_esmf.IN" + default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2swa${tmpl_suffix:-}.IN" ;; *) echo "FATAL ERROR: Unsupported UFSWM configuration for ${model_list}" diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 6d5d40a354..e9776d3eb6 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -135,13 +135,13 @@ for file in postxconfig-NT-GEFS-F00.txt postxconfig-NT-GEFS.txt postxconfig-NT-G postxconfig-NT-GFS-ANL.txt postxconfig-NT-GFS-F00.txt postxconfig-NT-GFS-FLUX-F00.txt \ postxconfig-NT-GFS.txt postxconfig-NT-GFS-FLUX.txt postxconfig-NT-GFS-GOES.txt \ postxconfig-NT-GFS-F00-TWO.txt postxconfig-NT-GFS-TWO.txt \ - params_grib2_tbl_new post_tag_gfs128 post_tag_gfs65 nam_micro_lookup.dat + params_grib2_tbl_new post_tag_gfs128 post_tag_gfs65 nam_micro_lookup.dat do ${LINK_OR_COPY} "${HOMEgfs}/sorc/upp.fd/parm/${file}" . done for file in optics_luts_DUST.dat optics_luts_DUST_nasa.dat optics_luts_NITR_nasa.dat \ optics_luts_SALT.dat optics_luts_SALT_nasa.dat optics_luts_SOOT.dat optics_luts_SOOT_nasa.dat \ - optics_luts_SUSO.dat optics_luts_SUSO_nasa.dat optics_luts_WASO.dat optics_luts_WASO_nasa.dat + optics_luts_SUSO.dat optics_luts_SUSO_nasa.dat optics_luts_WASO.dat optics_luts_WASO_nasa.dat do ${LINK_OR_COPY} "${HOMEgfs}/sorc/upp.fd/fix/chem/${file}" . done @@ -163,12 +163,18 @@ declare -a ufs_templates=("model_configure.IN" \ "MOM6_data_table.IN" \ "ice_in.IN" \ "ufs.configure.atm.IN" \ + "ufs.configure.atm_esmf.IN" \ "ufs.configure.atmaero.IN" \ - "ufs.configure.leapfrog_atm_wav.IN" \ + "ufs.configure.atmaero_esmf.IN" \ + "ufs.configure.s2s.IN" \ "ufs.configure.s2s_esmf.IN" \ + "ufs.configure.s2sa.IN" \ "ufs.configure.s2sa_esmf.IN" \ + "ufs.configure.s2sw.IN" \ "ufs.configure.s2sw_esmf.IN" \ - "ufs.configure.s2swa_esmf.IN" ) + "ufs.configure.s2swa.IN" \ + "ufs.configure.s2swa_esmf.IN" \ + "ufs.configure.leapfrog_atm_wav.IN" ) for file in "${ufs_templates[@]}"; do [[ -s "${file}" ]] && rm -f "${file}" ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/tests/parm/${file}" . From 9a96af9ab4289e6eb6203df056f27d1263f8bbf2 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Wed, 7 Feb 2024 13:16:24 -0500 Subject: [PATCH 03/14] update thread counts during job placement --- parm/config/gefs/config.resources | 4 ++-- parm/config/gfs/config.resources | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources index e300252d39..8736059eba 100644 --- a/parm/config/gefs/config.resources +++ b/parm/config/gefs/config.resources @@ -197,11 +197,11 @@ case ${step} in if [[ "${_CDUMP}" =~ "gfs" ]]; then declare -x "npe_${step}_gfs"="${NTASKS_TOT}" - declare -x "nth_${step}_gfs"=1 # ESMF handles threading for the UFS-weather-model + declare -x "nth_${step}_gfs"=${UFS_THREADS} declare -x "npe_node_${step}_gfs"="${npe_node_max}" else declare -x "npe_${step}"="${NTASKS_TOT}" - declare -x "nth_${step}"=1 # ESMF handles threading for the UFS-weather-model + declare -x "nth_${step}"=${UFS_THREADS} declare -x "npe_node_${step}"="${npe_node_max}" fi diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 69df8b724e..2715520670 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -535,6 +535,7 @@ case ${step} in ntasks_fv3=${ntasks_fv3_gfs} ntasks_quilt=${ntasks_quilt_gfs} nthreads_fv3=${nthreads_fv3_gfs} + nthreads_ufs=${nthreads_ufs_gfs} fi # Determine if using ESMF-managed threading or traditional threading From e8f18775fa2263f781f10974bdcb1bf83577a8ad Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Wed, 7 Feb 2024 13:44:35 -0500 Subject: [PATCH 04/14] update submodule pointer to ufs-weather-model --- .gitmodules | 4 +++- sorc/ufs_model.fd | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 3eb26fb0fe..3bc16df51b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,8 @@ [submodule "sorc/ufs_model.fd"] path = sorc/ufs_model.fd - url = https://github.com/ufs-community/ufs-weather-model + #url = https://github.com/ufs-community/ufs-weather-model + url = https://github.com/aerorahul/ufs-weather-model + branch = feature/ufs-threading ignore = dirty [submodule "sorc/wxflow"] path = sorc/wxflow diff --git a/sorc/ufs_model.fd b/sorc/ufs_model.fd index a837cfcb0c..d8a840cd98 160000 --- a/sorc/ufs_model.fd +++ b/sorc/ufs_model.fd @@ -1 +1 @@ -Subproject commit a837cfcb0c79803e13a1375a362d3d0b14b62320 +Subproject commit d8a840cd98266a3dc5a900750922ab952c98c814 From f21738b91cf533b5a13192a16e3102fe313e5ee4 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Thu, 7 Mar 2024 14:06:50 -0500 Subject: [PATCH 05/14] fix shellnorm errors --- parm/config/gefs/config.resources | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources index afcb6f1ed6..0be7e864a1 100644 --- a/parm/config/gefs/config.resources +++ b/parm/config/gefs/config.resources @@ -191,11 +191,11 @@ case ${step} in if [[ "${_CDUMP}" =~ "gfs" ]]; then declare -x "npe_${step}_gfs"="${NTASKS_TOT}" - declare -x "nth_${step}_gfs"=${UFS_THREADS} + declare -x "nth_${step}_gfs"="${UFS_THREADS}" declare -x "npe_node_${step}_gfs"="${npe_node_max}" else declare -x "npe_${step}"="${NTASKS_TOT}" - declare -x "nth_${step}"=${UFS_THREADS} + declare -x "nth_${step}"="${UFS_THREADS}" declare -x "npe_node_${step}"="${npe_node_max}" fi From 33d3eccf0687f6ef46f23e6f9e68751ff58d50d6 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Thu, 7 Mar 2024 14:20:26 -0500 Subject: [PATCH 06/14] update config.resources in the repo with the one used in test --- parm/config/gfs/config.resources | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index cf5665fa19..f63690a529 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -631,11 +631,11 @@ case ${step} in if [[ "${_CDUMP}" =~ "gfs" ]]; then declare -x "npe_${step}_gfs"="${NTASKS_TOT}" - declare -x "nth_${step}_gfs"=1 # ESMF handles threading for the UFS-weather-model + declare -x "nth_${step}_gfs"="${UFS_THREADS}" declare -x "npe_node_${step}_gfs"="${npe_node_max}" else declare -x "npe_${step}"="${NTASKS_TOT}" - declare -x "nth_${step}"=1 # ESMF handles threading for the UFS-weather-model + declare -x "nth_${step}"="${UFS_THREADS}" declare -x "npe_node_${step}"="${npe_node_max}" fi From e7e2b9a3d760537073221efbfc547682938a9058 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Wed, 13 Mar 2024 10:16:48 -0400 Subject: [PATCH 07/14] replace HOMEgfs/parm with PARMgfs to be consistent w/ gw --- parm/config/gefs/config.ufs | 14 +++++++------- parm/config/gfs/config.ufs | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/parm/config/gefs/config.ufs b/parm/config/gefs/config.ufs index 6e423865ee..b8695b6dbb 100644 --- a/parm/config/gefs/config.ufs +++ b/parm/config/gefs/config.ufs @@ -452,25 +452,25 @@ if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then fi case "${model_list}" in atm) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.atm${tmpl_suffix:-}.IN" + default_template="${PARMgfs}/ufs/ufs.configure.atm${tmpl_suffix:-}.IN" ;; atm.aero) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.atmaero${tmpl_suffix:-}.IN" + default_template="${PARMgfs}/ufs/ufs.configure.atmaero${tmpl_suffix:-}.IN" ;; atm.wave) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.leapfrog_atm_wav${tmpl_suffix:-}.IN" + default_template="${PARMgfs}/ufs/ufs.configure.leapfrog_atm_wav${tmpl_suffix:-}.IN" ;; atm.ocean.ice) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2s${tmpl_suffix:-}.IN" + default_template="${PARMgfs}/ufs/ufs.configure.s2s${tmpl_suffix:-}.IN" ;; atm.ocean.ice.aero) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2sa${tmpl_suffix:-}.IN" + default_template="${PARMgfs}/ufs/ufs.configure.s2sa${tmpl_suffix:-}.IN" ;; atm.ocean.ice.wave) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2sw${tmpl_suffix:-}.IN" + default_template="${PARMgfs}/ufs/ufs.configure.s2sw${tmpl_suffix:-}.IN" ;; atm.ocean.ice.wave.aero) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2swa${tmpl_suffix:-}.IN" + default_template="${PARMgfs}/ufs/ufs.configure.s2swa${tmpl_suffix:-}.IN" ;; *) echo "FATAL ERROR: Unsupported UFSWM configuration for ${model_list}" diff --git a/parm/config/gfs/config.ufs b/parm/config/gfs/config.ufs index 15366ab519..0db6f090a5 100644 --- a/parm/config/gfs/config.ufs +++ b/parm/config/gfs/config.ufs @@ -460,25 +460,25 @@ if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then fi case "${model_list}" in atm) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.atm${tmpl_suffix:-}.IN" + default_template="${PARMgfs}/ufs/ufs.configure.atm${tmpl_suffix:-}.IN" ;; atm.aero) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.atmaero${tmpl_suffix:-}.IN" + default_template="${PARMgfs}/ufs/ufs.configure.atmaero${tmpl_suffix:-}.IN" ;; atm.wave) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.leapfrog_atm_wav${tmpl_suffix:-}.IN" + default_template="${PARMgfs}/ufs/ufs.configure.leapfrog_atm_wav${tmpl_suffix:-}.IN" ;; atm.ocean.ice) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2s${tmpl_suffix:-}.IN" + default_template="${PARMgfs}/ufs/ufs.configure.s2s${tmpl_suffix:-}.IN" ;; atm.ocean.ice.aero) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2sa${tmpl_suffix:-}.IN" + default_template="${PARMgfs}/ufs/ufs.configure.s2sa${tmpl_suffix:-}.IN" ;; atm.ocean.ice.wave) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2sw${tmpl_suffix:-}.IN" + default_template="${PARMgfs}/ufs/ufs.configure.s2sw${tmpl_suffix:-}.IN" ;; atm.ocean.ice.wave.aero) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2swa${tmpl_suffix:-}.IN" + default_template="${PARMgfs}/ufs/ufs.configure.s2swa${tmpl_suffix:-}.IN" ;; *) echo "FATAL ERROR: Unsupported UFSWM configuration for ${model_list}" From e60ae211a7daf93188127a039f4ec99648d8fc68 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Wed, 13 Mar 2024 10:18:34 -0400 Subject: [PATCH 08/14] fix mixed-indent in .gitmodules --- .gitmodules | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitmodules b/.gitmodules index 0c89ebad72..cb2759bb86 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,9 @@ [submodule "sorc/ufs_model.fd"] - path = sorc/ufs_model.fd - #url = https://github.com/ufs-community/ufs-weather-model - url = https://github.com/aerorahul/ufs-weather-model + path = sorc/ufs_model.fd + #url = https://github.com/ufs-community/ufs-weather-model + url = https://github.com/aerorahul/ufs-weather-model branch = feature/ufs-threading - ignore = dirty + ignore = dirty [submodule "sorc/wxflow"] path = sorc/wxflow url = https://github.com/NOAA-EMC/wxflow From 65830b8b81f405cec8372540857b05dc03fce2c6 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Thu, 21 Mar 2024 09:44:44 -0400 Subject: [PATCH 09/14] update ufs-weather-model hash to 126db7a5 and in the authoritative repo --- .gitmodules | 4 +--- sorc/ufs_model.fd | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.gitmodules b/.gitmodules index cb2759bb86..5c9e569243 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,8 +1,6 @@ [submodule "sorc/ufs_model.fd"] path = sorc/ufs_model.fd - #url = https://github.com/ufs-community/ufs-weather-model - url = https://github.com/aerorahul/ufs-weather-model - branch = feature/ufs-threading + url = https://github.com/ufs-community/ufs-weather-model ignore = dirty [submodule "sorc/wxflow"] path = sorc/wxflow diff --git a/sorc/ufs_model.fd b/sorc/ufs_model.fd index a07bd60c07..126db7a514 160000 --- a/sorc/ufs_model.fd +++ b/sorc/ufs_model.fd @@ -1 +1 @@ -Subproject commit a07bd60c07be7222d0eea90e7459376304fbcafe +Subproject commit 126db7a5145319cb6a8b3d272215efcd5b88562f From c558151e24bd21b19816c600cd2dfc561838c0ec Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Thu, 21 Mar 2024 10:00:42 -0400 Subject: [PATCH 10/14] add a newline at the end of diag_table_aod --- parm/ufs/fv3/diag_table_aod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/ufs/fv3/diag_table_aod b/parm/ufs/fv3/diag_table_aod index 0de51b66d8..fd8aee1791 100644 --- a/parm/ufs/fv3/diag_table_aod +++ b/parm/ufs/fv3/diag_table_aod @@ -3,4 +3,4 @@ "gfs_phys", "SU_AOD_550", "su_aod550", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "BC_AOD_550", "bc_aod550", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "OC_AOD_550", "oc_aod550", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "SS_AOD_550", "ss_aod550", "fv3_history2d", "all", .false., "none", 2 \ No newline at end of file +"gfs_phys", "SS_AOD_550", "ss_aod550", "fv3_history2d", "all", .false., "none", 2 From 094c7da9a038e750901d762e67a8aa4ed6288d10 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Thu, 21 Mar 2024 10:16:54 -0400 Subject: [PATCH 11/14] increase memory for offline upp on wcoss2 for C768 --- parm/config/gfs/config.resources | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index f2e32d8b0f..9a15a9ad4a 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -706,9 +706,13 @@ case ${step} in "C48" | "C96") export npe_upp=${CASE:1} ;; - "C192" | "C384" | "C768") + "C192" | "C384") export npe_upp=120 - export memory_upp="48GB" + export memory_upp="96GB" + ;; + "C768") + export npe_upp=120 + if [[ ${machine} == "WCOSS2" ]]; then export memory_upp="480GB" ; fi ;; *) echo "FATAL ERROR: Resources not defined for job ${job} at resolution ${CASE}" From 9d8060e2fb369293a3d5b40ef25424f846d7b397 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Thu, 21 Mar 2024 10:20:06 -0400 Subject: [PATCH 12/14] Update sorc/link_workflow.sh Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> --- sorc/link_workflow.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 661c42ad12..9184e08432 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -177,7 +177,8 @@ declare -a ufs_templates=("model_configure.IN" \ "ufs.configure.s2sw_esmf.IN" \ "ufs.configure.s2swa.IN" \ "ufs.configure.s2swa_esmf.IN" \ - "ufs.configure.leapfrog_atm_wav.IN" ) + "ufs.configure.leapfrog_atm_wav.IN" \ + "ufs.configure.leapfrog_atm_wav_esmf.IN" ) for file in "${ufs_templates[@]}"; do [[ -s "${file}" ]] && rm -f "${file}" ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/tests/parm/${file}" . From ddd3ea05c1066b0e6305a598068437d53fc37376 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Thu, 21 Mar 2024 10:55:05 -0400 Subject: [PATCH 13/14] Update parm/config/gfs/config.resources --- parm/config/gfs/config.resources | 1 + 1 file changed, 1 insertion(+) diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 9a15a9ad4a..98fc3b2668 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -712,6 +712,7 @@ case ${step} in ;; "C768") export npe_upp=120 + export memory_upp="96GB" if [[ ${machine} == "WCOSS2" ]]; then export memory_upp="480GB" ; fi ;; *) From 668ede009b08644adaa76db4616c984f996e7afc Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Thu, 21 Mar 2024 12:13:13 -0400 Subject: [PATCH 14/14] replace reset w/ purge for orion --- ush/module-setup.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ush/module-setup.sh b/ush/module-setup.sh index 008ef7ee97..b66e3622d0 100755 --- a/ush/module-setup.sh +++ b/ush/module-setup.sh @@ -36,10 +36,10 @@ elif [[ ${MACHINE_ID} = orion* ]] ; then if ( ! eval module help > /dev/null 2>&1 ) ; then source /apps/lmod/lmod/init/bash fi - export LMOD_SYSTEM_DEFAULT_MODULES=contrib - set +u - module reset - set -u + #export LMOD_SYSTEM_DEFAULT_MODULES=git/2.28.0 # contrib has a lot of stuff we shouldn't put in MODULEPATH + #set +u + module purge # reset causes issues on Orion sometimes. + #set -u elif [[ ${MACHINE_ID} = s4* ]] ; then # We are on SSEC Wisconsin S4