From 102df72bc8f99e51f00d1755db33a923e462d3ed Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Tue, 6 Feb 2024 12:02:26 -0500 Subject: [PATCH 01/63] add toggle for ESMF/traditional threading for UFS --- parm/config/gefs/config.fcst | 1 + parm/config/gefs/config.resources | 16 +++++++++++++++- parm/config/gefs/config.ufs | 14 ++++++++++++++ parm/config/gfs/config.fcst | 1 + parm/config/gfs/config.resources | 15 ++++++++++++++- parm/config/gfs/config.ufs | 14 ++++++++++++++ scripts/exglobal_forecast.sh | 6 ++++++ 7 files changed, 65 insertions(+), 2 deletions(-) diff --git a/parm/config/gefs/config.fcst b/parm/config/gefs/config.fcst index 6a2a852e0b..224e961c16 100644 --- a/parm/config/gefs/config.fcst +++ b/parm/config/gefs/config.fcst @@ -37,6 +37,7 @@ if [[ "${DONST}" == "YES" ]]; then source "${EXPDIR}/config.nsst" fi +export USE_ESMF_THREADING="YES" # Toggle to use ESMF-managed threading or traditional threading in UFSWM export esmf_profile=".false." export esmf_logkind="ESMF_LOGKIND_MULTI_ON_ERROR" #Options: ESMF_LOGKIND_MULTI_ON_ERROR, ESMF_LOGKIND_MULTI, ESMF_LOGKIND_NONE diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources index 36b70aecb8..e300252d39 100644 --- a/parm/config/gefs/config.resources +++ b/parm/config/gefs/config.resources @@ -107,6 +107,20 @@ case ${step} in ntasks_fv3=${ntasks_fv3_gfs} ntasks_quilt=${ntasks_quilt_gfs} nthreads_fv3=${nthreads_fv3_gfs} + nthreads_ufs=${nthreads_ufs_gfs} + fi + + # Determine if using ESMF-managed threading or traditional threading + # If using traditional threading, set them to 1 + if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then + export UFS_THREADS=1 + else # traditional threading + export UFS_THREADS=${nthreads_ufs:-1} + nthreads_fv3=1 + nthreads_mediator=1 + [[ "${DO_WAVE}" == "YES" ]] && nthreads_ww3=1 + [[ "${DO_OCN}" == "YES" ]] && nthreads_mom6=1 + [[ "${DO_ICE}" == "YES" ]] && nthreads_cice6=1 fi # PETS for the atmosphere dycore @@ -239,4 +253,4 @@ case ${step} in esac -echo "END: config.resources" \ No newline at end of file +echo "END: config.resources" diff --git a/parm/config/gefs/config.ufs b/parm/config/gefs/config.ufs index 2031d0b538..76c2060f8d 100644 --- a/parm/config/gefs/config.ufs +++ b/parm/config/gefs/config.ufs @@ -126,6 +126,8 @@ case "${fv3_res}" in export layout_y_gfs=1 export nthreads_fv3=1 export nthreads_fv3_gfs=1 + export nthreads_ufs=1 + export nthreads_ufs_gfs=1 export cdmbgwd="0.071,2.1,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="40.0,1.77,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=6.0e-3 # setting for UGWPv1 non-stationary GWD @@ -142,6 +144,8 @@ case "${fv3_res}" in export layout_y_gfs=2 export nthreads_fv3=1 export nthreads_fv3_gfs=1 + export nthreads_ufs=1 + export nthreads_ufs_gfs=1 export cdmbgwd="0.14,1.8,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="20.0,2.5,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=3.0e-3 # setting for UGWPv1 non-stationary GWD @@ -158,6 +162,8 @@ case "${fv3_res}" in export layout_y_gfs=6 export nthreads_fv3=1 export nthreads_fv3_gfs=2 + export nthreads_ufs=1 + export nthreads_ufs_gfs=2 export cdmbgwd="0.23,1.5,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="10.0,3.5,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=1.5e-3 # setting for UGWPv1 non-stationary GWD @@ -174,6 +180,8 @@ case "${fv3_res}" in export layout_y_gfs=8 export nthreads_fv3=1 export nthreads_fv3_gfs=2 + export nthreads_ufs=1 + export nthreads_ufs_gfs=2 export cdmbgwd="1.1,0.72,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="5.0,5.0,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=0.8e-3 # setting for UGWPv1 non-stationary GWD @@ -190,6 +198,8 @@ case "${fv3_res}" in export layout_y_gfs=16 export nthreads_fv3=4 export nthreads_fv3_gfs=4 + export nthreads_ufs=4 + export nthreads_ufs_gfs=4 export cdmbgwd="4.0,0.15,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="2.5,7.5,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=0.5e-3 # setting for UGWPv1 non-stationary GWD @@ -206,6 +216,8 @@ case "${fv3_res}" in export layout_y_gfs=16 export nthreads_fv3=4 export nthreads_fv3_gfs=4 + export nthreads_ufs=4 + export nthreads_ufs_gfs=4 export cdmbgwd="4.0,0.10,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="1.67,8.8,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=0.35e-3 # setting for UGWPv1 non-stationary GWD @@ -222,6 +234,8 @@ case "${fv3_res}" in export layout_y_gfs=32 export nthreads_fv3=4 export nthreads_fv3_gfs=4 + export nthreads_ufs=4 + export nthreads_ufs_gfs=4 export cdmbgwd="4.0,0.05,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="0.625,14.1,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=0.13e-3 # setting for UGWPv1 non-stationary GWD diff --git a/parm/config/gfs/config.fcst b/parm/config/gfs/config.fcst index d2e2664e9c..4a6f932ac9 100644 --- a/parm/config/gfs/config.fcst +++ b/parm/config/gfs/config.fcst @@ -49,6 +49,7 @@ if [[ "${DONST}" == "YES" ]]; then source "${EXPDIR}/config.nsst" fi +export USE_ESMF_THREADING="YES" # Toggle to use ESMF-managed threading or traditional threading in UFSWM export esmf_profile=".false." export esmf_logkind="ESMF_LOGKIND_MULTI_ON_ERROR" #Options: ESMF_LOGKIND_MULTI_ON_ERROR, ESMF_LOGKIND_MULTI, ESMF_LOGKIND_NONE diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 1dcf757de2..69df8b724e 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -537,6 +537,19 @@ case ${step} in nthreads_fv3=${nthreads_fv3_gfs} fi + # Determine if using ESMF-managed threading or traditional threading + # If using traditional threading, set them to 1 + if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then + export UFS_THREADS=1 + else # traditional threading + export UFS_THREADS=${nthreads_ufs:-1} + nthreads_fv3=1 + nthreads_mediator=1 + [[ "${DO_WAVE}" == "YES" ]] && nthreads_ww3=1 + [[ "${DO_OCN}" == "YES" ]] && nthreads_mom6=1 + [[ "${DO_ICE}" == "YES" ]] && nthreads_cice6=1 + fi + # PETS for the atmosphere dycore (( FV3PETS = ntasks_fv3 * nthreads_fv3 )) echo "FV3 using (nthreads, PETS) = (${nthreads_fv3}, ${FV3PETS})" @@ -926,7 +939,7 @@ case ${step} in export nth_eupd=4 fi ;; - *) + *) echo "FATAL ERROR: Resources not defined for job ${job} at resolution ${CASE}" exit 4 ;; diff --git a/parm/config/gfs/config.ufs b/parm/config/gfs/config.ufs index 0a59da47ca..0a8f2f9d15 100644 --- a/parm/config/gfs/config.ufs +++ b/parm/config/gfs/config.ufs @@ -126,6 +126,8 @@ case "${fv3_res}" in export layout_y_gfs=1 export nthreads_fv3=1 export nthreads_fv3_gfs=1 + export nthreads_ufs=1 + export nthreads_ufs_gfs=1 export cdmbgwd="0.071,2.1,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="40.0,1.77,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=6.0e-3 # setting for UGWPv1 non-stationary GWD @@ -142,6 +144,8 @@ case "${fv3_res}" in export layout_y_gfs=2 export nthreads_fv3=1 export nthreads_fv3_gfs=1 + export nthreads_ufs=1 + export nthreads_ufs_gfs=1 export cdmbgwd="0.14,1.8,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="20.0,2.5,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=3.0e-3 # setting for UGWPv1 non-stationary GWD @@ -158,6 +162,8 @@ case "${fv3_res}" in export layout_y_gfs=6 export nthreads_fv3=1 export nthreads_fv3_gfs=2 + export nthreads_ufs=1 + export nthreads_ufs_gfs=2 export cdmbgwd="0.23,1.5,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="10.0,3.5,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=1.5e-3 # setting for UGWPv1 non-stationary GWD @@ -174,6 +180,8 @@ case "${fv3_res}" in export layout_y_gfs=8 export nthreads_fv3=2 export nthreads_fv3_gfs=2 + export nthreads_ufs=2 + export nthreads_ufs_gfs=2 export cdmbgwd="1.1,0.72,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="5.0,5.0,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=0.8e-3 # setting for UGWPv1 non-stationary GWD @@ -190,6 +198,8 @@ case "${fv3_res}" in export layout_y_gfs=16 export nthreads_fv3=4 export nthreads_fv3_gfs=4 + export nthreads_ufs=4 + export nthreads_ufs_gfs=4 export cdmbgwd="4.0,0.15,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="2.5,7.5,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=0.5e-3 # setting for UGWPv1 non-stationary GWD @@ -206,6 +216,8 @@ case "${fv3_res}" in export layout_y_gfs=16 export nthreads_fv3=4 export nthreads_fv3_gfs=4 + export nthreads_ufs=4 + export nthreads_ufs_gfs=4 export cdmbgwd="4.0,0.10,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="1.67,8.8,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=0.35e-3 # setting for UGWPv1 non-stationary GWD @@ -222,6 +234,8 @@ case "${fv3_res}" in export layout_y_gfs=32 export nthreads_fv3=4 export nthreads_fv3_gfs=4 + export nthreads_ufs=4 + export nthreads_ufs_gfs=4 export cdmbgwd="4.0,0.05,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="0.625,14.1,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=0.13e-3 # setting for UGWPv1 non-stationary GWD diff --git a/scripts/exglobal_forecast.sh b/scripts/exglobal_forecast.sh index eebc9e59c3..276d4c8a7c 100755 --- a/scripts/exglobal_forecast.sh +++ b/scripts/exglobal_forecast.sh @@ -146,6 +146,12 @@ if [[ "${esmf_profile:-}" = ".true." ]]; then export ESMF_RUNTIME_PROFILE_OUTPUT=SUMMARY fi +if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then + unset OMP_NUM_THREADS +else + export OMP_NUM_THREADS=${UFS_THREADS:-1} +fi + ${NCP} "${FCSTEXECDIR}/${FCSTEXEC}" "${DATA}/" ${APRUN_UFS} "${DATA}/${FCSTEXEC}" 1>&1 2>&2 export ERR=$? From 1d71b231a48070f1dbcf7188b7309ef02b6a8786 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Wed, 7 Feb 2024 12:55:55 -0500 Subject: [PATCH 02/63] choose the appropriate template based on threading choice for ufswm --- parm/config/gefs/config.fcst | 3 ++- parm/config/gefs/config.ufs | 17 ++++++++++------- parm/config/gfs/config.fcst | 3 ++- parm/config/gfs/config.ufs | 17 ++++++++++------- sorc/link_workflow.sh | 14 ++++++++++---- 5 files changed, 34 insertions(+), 20 deletions(-) diff --git a/parm/config/gefs/config.fcst b/parm/config/gefs/config.fcst index 224e961c16..9f00739957 100644 --- a/parm/config/gefs/config.fcst +++ b/parm/config/gefs/config.fcst @@ -5,6 +5,8 @@ echo "BEGIN: config.fcst" +export USE_ESMF_THREADING="YES" # Toggle to use ESMF-managed threading or traditional threading in UFSWM + # Turn off waves if not used for this CDUMP case ${WAVE_CDUMP} in both | "${CDUMP/enkf}" ) ;; # Don't change @@ -37,7 +39,6 @@ if [[ "${DONST}" == "YES" ]]; then source "${EXPDIR}/config.nsst" fi -export USE_ESMF_THREADING="YES" # Toggle to use ESMF-managed threading or traditional threading in UFSWM export esmf_profile=".false." export esmf_logkind="ESMF_LOGKIND_MULTI_ON_ERROR" #Options: ESMF_LOGKIND_MULTI_ON_ERROR, ESMF_LOGKIND_MULTI, ESMF_LOGKIND_NONE diff --git a/parm/config/gefs/config.ufs b/parm/config/gefs/config.ufs index 76c2060f8d..deedd2525e 100644 --- a/parm/config/gefs/config.ufs +++ b/parm/config/gefs/config.ufs @@ -495,27 +495,30 @@ fi # Set the name of the UFS (previously nems) configure template to use # Default ufs.configure templates for supported model configurations +if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then + tmpl_suffix="_esmf" +fi case "${model_list}" in atm) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.atm.IN" + default_template="${HOMEgfs}/parm/ufs/ufs.configure.atm${tmpl_suffix:-}.IN" ;; atm.aero) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.atmaero.IN" + default_template="${HOMEgfs}/parm/ufs/ufs.configure.atmaero${tmpl_suffix:-}.IN" ;; atm.wave) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.leapfrog_atm_wav.IN" + default_template="${HOMEgfs}/parm/ufs/ufs.configure.leapfrog_atm_wav${tmpl_suffix:-}.IN" ;; atm.ocean.ice) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2s_esmf.IN" + default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2s${tmpl_suffix:-}.IN" ;; atm.ocean.ice.aero) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2sa_esmf.IN" + default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2sa${tmpl_suffix:-}.IN" ;; atm.ocean.ice.wave) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2sw_esmf.IN" + default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2sw${tmpl_suffix:-}.IN" ;; atm.ocean.ice.wave.aero) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2swa_esmf.IN" + default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2swa${tmpl_suffix:-}.IN" ;; *) echo "FATAL ERROR: Unsupported UFSWM configuration for ${model_list}" diff --git a/parm/config/gfs/config.fcst b/parm/config/gfs/config.fcst index 4a6f932ac9..fbcb4029b6 100644 --- a/parm/config/gfs/config.fcst +++ b/parm/config/gfs/config.fcst @@ -5,6 +5,8 @@ echo "BEGIN: config.fcst" +export USE_ESMF_THREADING="YES" # Toggle to use ESMF-managed threading or traditional threading in UFSWM + # Turn off waves if not used for this CDUMP case ${WAVE_CDUMP} in both | "${CDUMP/enkf}" ) ;; # Don't change @@ -49,7 +51,6 @@ if [[ "${DONST}" == "YES" ]]; then source "${EXPDIR}/config.nsst" fi -export USE_ESMF_THREADING="YES" # Toggle to use ESMF-managed threading or traditional threading in UFSWM export esmf_profile=".false." export esmf_logkind="ESMF_LOGKIND_MULTI_ON_ERROR" #Options: ESMF_LOGKIND_MULTI_ON_ERROR, ESMF_LOGKIND_MULTI, ESMF_LOGKIND_NONE diff --git a/parm/config/gfs/config.ufs b/parm/config/gfs/config.ufs index 0a8f2f9d15..eff25ea3a7 100644 --- a/parm/config/gfs/config.ufs +++ b/parm/config/gfs/config.ufs @@ -495,27 +495,30 @@ fi # Set the name of the UFS (previously nems) configure template to use # Default ufs.configure templates for supported model configurations +if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then + tmpl_suffix="_esmf" +fi case "${model_list}" in atm) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.atm.IN" + default_template="${HOMEgfs}/parm/ufs/ufs.configure.atm${tmpl_suffix:-}.IN" ;; atm.aero) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.atmaero.IN" + default_template="${HOMEgfs}/parm/ufs/ufs.configure.atmaero${tmpl_suffix:-}.IN" ;; atm.wave) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.leapfrog_atm_wav.IN" + default_template="${HOMEgfs}/parm/ufs/ufs.configure.leapfrog_atm_wav${tmpl_suffix:-}.IN" ;; atm.ocean.ice) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2s_esmf.IN" + default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2s${tmpl_suffix:-}.IN" ;; atm.ocean.ice.aero) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2sa_esmf.IN" + default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2sa${tmpl_suffix:-}.IN" ;; atm.ocean.ice.wave) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2sw_esmf.IN" + default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2sw${tmpl_suffix:-}.IN" ;; atm.ocean.ice.wave.aero) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2swa_esmf.IN" + default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2swa${tmpl_suffix:-}.IN" ;; *) echo "FATAL ERROR: Unsupported UFSWM configuration for ${model_list}" diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 6d5d40a354..e9776d3eb6 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -135,13 +135,13 @@ for file in postxconfig-NT-GEFS-F00.txt postxconfig-NT-GEFS.txt postxconfig-NT-G postxconfig-NT-GFS-ANL.txt postxconfig-NT-GFS-F00.txt postxconfig-NT-GFS-FLUX-F00.txt \ postxconfig-NT-GFS.txt postxconfig-NT-GFS-FLUX.txt postxconfig-NT-GFS-GOES.txt \ postxconfig-NT-GFS-F00-TWO.txt postxconfig-NT-GFS-TWO.txt \ - params_grib2_tbl_new post_tag_gfs128 post_tag_gfs65 nam_micro_lookup.dat + params_grib2_tbl_new post_tag_gfs128 post_tag_gfs65 nam_micro_lookup.dat do ${LINK_OR_COPY} "${HOMEgfs}/sorc/upp.fd/parm/${file}" . done for file in optics_luts_DUST.dat optics_luts_DUST_nasa.dat optics_luts_NITR_nasa.dat \ optics_luts_SALT.dat optics_luts_SALT_nasa.dat optics_luts_SOOT.dat optics_luts_SOOT_nasa.dat \ - optics_luts_SUSO.dat optics_luts_SUSO_nasa.dat optics_luts_WASO.dat optics_luts_WASO_nasa.dat + optics_luts_SUSO.dat optics_luts_SUSO_nasa.dat optics_luts_WASO.dat optics_luts_WASO_nasa.dat do ${LINK_OR_COPY} "${HOMEgfs}/sorc/upp.fd/fix/chem/${file}" . done @@ -163,12 +163,18 @@ declare -a ufs_templates=("model_configure.IN" \ "MOM6_data_table.IN" \ "ice_in.IN" \ "ufs.configure.atm.IN" \ + "ufs.configure.atm_esmf.IN" \ "ufs.configure.atmaero.IN" \ - "ufs.configure.leapfrog_atm_wav.IN" \ + "ufs.configure.atmaero_esmf.IN" \ + "ufs.configure.s2s.IN" \ "ufs.configure.s2s_esmf.IN" \ + "ufs.configure.s2sa.IN" \ "ufs.configure.s2sa_esmf.IN" \ + "ufs.configure.s2sw.IN" \ "ufs.configure.s2sw_esmf.IN" \ - "ufs.configure.s2swa_esmf.IN" ) + "ufs.configure.s2swa.IN" \ + "ufs.configure.s2swa_esmf.IN" \ + "ufs.configure.leapfrog_atm_wav.IN" ) for file in "${ufs_templates[@]}"; do [[ -s "${file}" ]] && rm -f "${file}" ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/tests/parm/${file}" . From 9a96af9ab4289e6eb6203df056f27d1263f8bbf2 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Wed, 7 Feb 2024 13:16:24 -0500 Subject: [PATCH 03/63] update thread counts during job placement --- parm/config/gefs/config.resources | 4 ++-- parm/config/gfs/config.resources | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources index e300252d39..8736059eba 100644 --- a/parm/config/gefs/config.resources +++ b/parm/config/gefs/config.resources @@ -197,11 +197,11 @@ case ${step} in if [[ "${_CDUMP}" =~ "gfs" ]]; then declare -x "npe_${step}_gfs"="${NTASKS_TOT}" - declare -x "nth_${step}_gfs"=1 # ESMF handles threading for the UFS-weather-model + declare -x "nth_${step}_gfs"=${UFS_THREADS} declare -x "npe_node_${step}_gfs"="${npe_node_max}" else declare -x "npe_${step}"="${NTASKS_TOT}" - declare -x "nth_${step}"=1 # ESMF handles threading for the UFS-weather-model + declare -x "nth_${step}"=${UFS_THREADS} declare -x "npe_node_${step}"="${npe_node_max}" fi diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 69df8b724e..2715520670 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -535,6 +535,7 @@ case ${step} in ntasks_fv3=${ntasks_fv3_gfs} ntasks_quilt=${ntasks_quilt_gfs} nthreads_fv3=${nthreads_fv3_gfs} + nthreads_ufs=${nthreads_ufs_gfs} fi # Determine if using ESMF-managed threading or traditional threading From e8f18775fa2263f781f10974bdcb1bf83577a8ad Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Wed, 7 Feb 2024 13:44:35 -0500 Subject: [PATCH 04/63] update submodule pointer to ufs-weather-model --- .gitmodules | 4 +++- sorc/ufs_model.fd | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 3eb26fb0fe..3bc16df51b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,8 @@ [submodule "sorc/ufs_model.fd"] path = sorc/ufs_model.fd - url = https://github.com/ufs-community/ufs-weather-model + #url = https://github.com/ufs-community/ufs-weather-model + url = https://github.com/aerorahul/ufs-weather-model + branch = feature/ufs-threading ignore = dirty [submodule "sorc/wxflow"] path = sorc/wxflow diff --git a/sorc/ufs_model.fd b/sorc/ufs_model.fd index a837cfcb0c..d8a840cd98 160000 --- a/sorc/ufs_model.fd +++ b/sorc/ufs_model.fd @@ -1 +1 @@ -Subproject commit a837cfcb0c79803e13a1375a362d3d0b14b62320 +Subproject commit d8a840cd98266a3dc5a900750922ab952c98c814 From 82420554c1576598e4c4ed9ebb7130448c292362 Mon Sep 17 00:00:00 2001 From: HenryWinterbottom-NOAA Date: Tue, 27 Feb 2024 17:02:30 +0000 Subject: [PATCH 05/63] Updating/debugging gfs_utils. --- .gitmodules | 3 ++- modulefiles/module_gwsetup.hera.lua | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 3eb26fb0fe..63adea28f7 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,7 +7,8 @@ url = https://github.com/NOAA-EMC/wxflow [submodule "sorc/gfs_utils.fd"] path = sorc/gfs_utils.fd - url = https://github.com/NOAA-EMC/gfs-utils + url = https://github.com/HenryWinterbottom-NOAA/gfs-utils + branch = feature/gwdev_issue_2329 [submodule "sorc/ufs_utils.fd"] path = sorc/ufs_utils.fd url = https://github.com/ufs-community/UFS_UTILS.git diff --git a/modulefiles/module_gwsetup.hera.lua b/modulefiles/module_gwsetup.hera.lua index 3e8bf2d7f8..31cb1d9f4f 100644 --- a/modulefiles/module_gwsetup.hera.lua +++ b/modulefiles/module_gwsetup.hera.lua @@ -4,7 +4,8 @@ Load environment to run GFS workflow setup scripts on Hera load(pathJoin("rocoto")) -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev/install/modulefiles/Core") +--prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev/install/modulefiles/Core") +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev-rocky8/install/modulefiles/Core") local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" local python_ver=os.getenv("python_ver") or "3.11.6" From febabd9f3bfa4ee4212da166d760f0b84d1fe6d8 Mon Sep 17 00:00:00 2001 From: HenryWinterbottom-NOAA Date: Tue, 27 Feb 2024 18:10:59 +0000 Subject: [PATCH 06/63] Updated submodule configuration --- .gitmodules | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 63adea28f7..cc6745b60e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -11,7 +11,8 @@ branch = feature/gwdev_issue_2329 [submodule "sorc/ufs_utils.fd"] path = sorc/ufs_utils.fd - url = https://github.com/ufs-community/UFS_UTILS.git + url = https://github.com/HenryWinterbottom-NOAA/UFS_UTILS + branch = feature/gwdev_issue_2329 [submodule "sorc/verif-global.fd"] path = sorc/verif-global.fd url = https://github.com/NOAA-EMC/EMC_verif-global.git From 5a4b36b3b436f8f83fbbb04436dd3f6cc7930ccc Mon Sep 17 00:00:00 2001 From: HenryWinterbottom-NOAA Date: Wed, 28 Feb 2024 17:52:50 +0000 Subject: [PATCH 07/63] Removed ufs_utils module; will be updated with test branch. --- .gitmodules | 8 ++++---- sorc/ufs_utils.fd | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) delete mode 160000 sorc/ufs_utils.fd diff --git a/.gitmodules b/.gitmodules index cc6745b60e..feddb391d8 100644 --- a/.gitmodules +++ b/.gitmodules @@ -9,10 +9,10 @@ path = sorc/gfs_utils.fd url = https://github.com/HenryWinterbottom-NOAA/gfs-utils branch = feature/gwdev_issue_2329 -[submodule "sorc/ufs_utils.fd"] - path = sorc/ufs_utils.fd - url = https://github.com/HenryWinterbottom-NOAA/UFS_UTILS - branch = feature/gwdev_issue_2329 +#[submodule "sorc/ufs_utils.fd"] +# path = sorc/ufs_utils.fd +# url = https://github.com/HenryWinterbottom-NOAA/UFS_UTILS +# branch = feature/gwdev_issue_2329 [submodule "sorc/verif-global.fd"] path = sorc/verif-global.fd url = https://github.com/NOAA-EMC/EMC_verif-global.git diff --git a/sorc/ufs_utils.fd b/sorc/ufs_utils.fd deleted file mode 160000 index 47956a96a7..0000000000 --- a/sorc/ufs_utils.fd +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 47956a96a7dc14e33fbccbb7fe74422f9bf542bf From c779301b38178712c9e22027e1b33ee0cc03f1bb Mon Sep 17 00:00:00 2001 From: HenryWinterbottom-NOAA Date: Wed, 28 Feb 2024 18:08:45 +0000 Subject: [PATCH 08/63] Fixed broken submodule load; add ufs_utils.fd back to repo with correct feature branch. --- .gitmodules | 8 ++++---- sorc/ufs_utils.fd | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) create mode 160000 sorc/ufs_utils.fd diff --git a/.gitmodules b/.gitmodules index feddb391d8..ec4525166d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -9,10 +9,6 @@ path = sorc/gfs_utils.fd url = https://github.com/HenryWinterbottom-NOAA/gfs-utils branch = feature/gwdev_issue_2329 -#[submodule "sorc/ufs_utils.fd"] -# path = sorc/ufs_utils.fd -# url = https://github.com/HenryWinterbottom-NOAA/UFS_UTILS -# branch = feature/gwdev_issue_2329 [submodule "sorc/verif-global.fd"] path = sorc/verif-global.fd url = https://github.com/NOAA-EMC/EMC_verif-global.git @@ -28,3 +24,7 @@ [submodule "sorc/gsi_monitor.fd"] path = sorc/gsi_monitor.fd url = https://github.com/NOAA-EMC/GSI-Monitor.git +[submodule "sorc/ufs_utils.fd"] + path = sorc/ufs_utils.fd + url = https://github.com/HenryWinterbottom-NOAA/UFS_UTILS + branch = feature/gwdev_issue_2329 diff --git a/sorc/ufs_utils.fd b/sorc/ufs_utils.fd new file mode 160000 index 0000000000..580f5ba80d --- /dev/null +++ b/sorc/ufs_utils.fd @@ -0,0 +1 @@ +Subproject commit 580f5ba80d78666321ba3cc08485f43cace8e655 From b7efb5f8af4c085d2430e283fd1d32126fd70d47 Mon Sep 17 00:00:00 2001 From: HenryWinterbottom-NOAA Date: Mon, 4 Mar 2024 16:53:11 +0000 Subject: [PATCH 09/63] Removed submodule. --- .gitmodules | 8 ++++---- sorc/ufs_utils.fd | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) delete mode 160000 sorc/ufs_utils.fd diff --git a/.gitmodules b/.gitmodules index ec4525166d..7058155db5 100644 --- a/.gitmodules +++ b/.gitmodules @@ -24,7 +24,7 @@ [submodule "sorc/gsi_monitor.fd"] path = sorc/gsi_monitor.fd url = https://github.com/NOAA-EMC/GSI-Monitor.git -[submodule "sorc/ufs_utils.fd"] - path = sorc/ufs_utils.fd - url = https://github.com/HenryWinterbottom-NOAA/UFS_UTILS - branch = feature/gwdev_issue_2329 +#[submodule "sorc/ufs_utils.fd"] +# path = sorc/ufs_utils.fd +# url = https://github.com/HenryWinterbottom-NOAA/UFS_UTILS +# branch = feature/gwdev_issue_2329 diff --git a/sorc/ufs_utils.fd b/sorc/ufs_utils.fd deleted file mode 160000 index 580f5ba80d..0000000000 --- a/sorc/ufs_utils.fd +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 580f5ba80d78666321ba3cc08485f43cace8e655 From 147c6813acf1188d02e6b808ed3477afce534181 Mon Sep 17 00:00:00 2001 From: HenryWinterbottom-NOAA Date: Mon, 4 Mar 2024 16:56:38 +0000 Subject: [PATCH 10/63] Updates for UFS_UTILS. --- .gitmodules | 4 ++++ sorc/ufs_utils.fd | 1 + 2 files changed, 5 insertions(+) create mode 160000 sorc/ufs_utils.fd diff --git a/.gitmodules b/.gitmodules index 7058155db5..e7198be86e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -28,3 +28,7 @@ # path = sorc/ufs_utils.fd # url = https://github.com/HenryWinterbottom-NOAA/UFS_UTILS # branch = feature/gwdev_issue_2329 +[submodule "sorc/ufs_utils.fd"] + path = sorc/ufs_utils.fd + url = https://github.com/HenryWinterbottom-NOAA/UFS_UTILS + branch = feature/gwdev_issue_2329 diff --git a/sorc/ufs_utils.fd b/sorc/ufs_utils.fd new file mode 160000 index 0000000000..580f5ba80d --- /dev/null +++ b/sorc/ufs_utils.fd @@ -0,0 +1 @@ +Subproject commit 580f5ba80d78666321ba3cc08485f43cace8e655 From b7580c26f6b96e110a387035fc21133e888655eb Mon Sep 17 00:00:00 2001 From: HenryWinterbottom-NOAA Date: Mon, 4 Mar 2024 17:15:37 +0000 Subject: [PATCH 11/63] Removed submodule. --- .gitmodules | 7 ------- sorc/gsi_utils.fd | 1 - 2 files changed, 8 deletions(-) delete mode 160000 sorc/gsi_utils.fd diff --git a/.gitmodules b/.gitmodules index e7198be86e..8f9b3be2da 100644 --- a/.gitmodules +++ b/.gitmodules @@ -18,16 +18,9 @@ [submodule "sorc/gdas.cd"] path = sorc/gdas.cd url = https://github.com/NOAA-EMC/GDASApp.git -[submodule "sorc/gsi_utils.fd"] - path = sorc/gsi_utils.fd - url = https://github.com/NOAA-EMC/GSI-Utils.git [submodule "sorc/gsi_monitor.fd"] path = sorc/gsi_monitor.fd url = https://github.com/NOAA-EMC/GSI-Monitor.git -#[submodule "sorc/ufs_utils.fd"] -# path = sorc/ufs_utils.fd -# url = https://github.com/HenryWinterbottom-NOAA/UFS_UTILS -# branch = feature/gwdev_issue_2329 [submodule "sorc/ufs_utils.fd"] path = sorc/ufs_utils.fd url = https://github.com/HenryWinterbottom-NOAA/UFS_UTILS diff --git a/sorc/gsi_utils.fd b/sorc/gsi_utils.fd deleted file mode 160000 index 55abe58825..0000000000 --- a/sorc/gsi_utils.fd +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 55abe588252ec6f39047d54a14727cf59f7f6688 From 784d3bc60ccccd2197fb8e8db95a706d839c7f54 Mon Sep 17 00:00:00 2001 From: HenryWinterbottom-NOAA Date: Mon, 4 Mar 2024 17:19:53 +0000 Subject: [PATCH 12/63] Rocky 8 updates. --- .gitmodules | 4 ++++ sorc/gsi_utils.fd | 1 + 2 files changed, 5 insertions(+) create mode 160000 sorc/gsi_utils.fd diff --git a/.gitmodules b/.gitmodules index 8f9b3be2da..7a3273fef8 100644 --- a/.gitmodules +++ b/.gitmodules @@ -25,3 +25,7 @@ path = sorc/ufs_utils.fd url = https://github.com/HenryWinterbottom-NOAA/UFS_UTILS branch = feature/gwdev_issue_2329 +[submodule "sorc/gsi_utils.fd"] + path = sorc/gsi_utils.fd + url = https://github.com/HenryWinterbottom-NOAA/GSI-utils + branch = feature/gwdev_issue_2329 diff --git a/sorc/gsi_utils.fd b/sorc/gsi_utils.fd new file mode 160000 index 0000000000..c52f065e58 --- /dev/null +++ b/sorc/gsi_utils.fd @@ -0,0 +1 @@ +Subproject commit c52f065e587951d6f6b024695983eb3a7769d6ea From 1648dca0421e5a448d6c22bad85c324688f2e59c Mon Sep 17 00:00:00 2001 From: HenryWinterbottom-NOAA Date: Mon, 4 Mar 2024 18:20:21 +0000 Subject: [PATCH 13/63] Removed submodule. --- .gitmodules | 3 --- sorc/verif-global.fd | 1 - 2 files changed, 4 deletions(-) delete mode 160000 sorc/verif-global.fd diff --git a/.gitmodules b/.gitmodules index 7a3273fef8..fdd4df1050 100644 --- a/.gitmodules +++ b/.gitmodules @@ -9,9 +9,6 @@ path = sorc/gfs_utils.fd url = https://github.com/HenryWinterbottom-NOAA/gfs-utils branch = feature/gwdev_issue_2329 -[submodule "sorc/verif-global.fd"] - path = sorc/verif-global.fd - url = https://github.com/NOAA-EMC/EMC_verif-global.git [submodule "sorc/gsi_enkf.fd"] path = sorc/gsi_enkf.fd url = https://github.com/NOAA-EMC/GSI.git diff --git a/sorc/verif-global.fd b/sorc/verif-global.fd deleted file mode 160000 index bd1c8f62a1..0000000000 --- a/sorc/verif-global.fd +++ /dev/null @@ -1 +0,0 @@ -Subproject commit bd1c8f62a1878051e34ff7c6f6a4dd290381f1ef From 2fe0bdd1a31a762f9c484206eabba1139f8b992c Mon Sep 17 00:00:00 2001 From: HenryWinterbottom-NOAA Date: Wed, 6 Mar 2024 15:37:54 +0000 Subject: [PATCH 14/63] Removed submodule. --- .gitmodules | 3 --- sorc/gsi_enkf.fd | 1 - 2 files changed, 4 deletions(-) delete mode 160000 sorc/gsi_enkf.fd diff --git a/.gitmodules b/.gitmodules index fdd4df1050..67a4a64be0 100644 --- a/.gitmodules +++ b/.gitmodules @@ -9,9 +9,6 @@ path = sorc/gfs_utils.fd url = https://github.com/HenryWinterbottom-NOAA/gfs-utils branch = feature/gwdev_issue_2329 -[submodule "sorc/gsi_enkf.fd"] - path = sorc/gsi_enkf.fd - url = https://github.com/NOAA-EMC/GSI.git [submodule "sorc/gdas.cd"] path = sorc/gdas.cd url = https://github.com/NOAA-EMC/GDASApp.git diff --git a/sorc/gsi_enkf.fd b/sorc/gsi_enkf.fd deleted file mode 160000 index 74ac594211..0000000000 --- a/sorc/gsi_enkf.fd +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 74ac5942118d2a83ca84d3a629ec3aaffdb36fc5 From a9a83b141f752b8504106d9cc1cd549749301f6e Mon Sep 17 00:00:00 2001 From: HenryWinterbottom-NOAA Date: Wed, 6 Mar 2024 15:42:31 +0000 Subject: [PATCH 15/63] Removed incorrect submodule. --- .gitmodules | 3 +++ sorc/sorc/gsi_enkf.fd | 1 + 2 files changed, 4 insertions(+) create mode 160000 sorc/sorc/gsi_enkf.fd diff --git a/.gitmodules b/.gitmodules index 67a4a64be0..3fa8b9cd58 100644 --- a/.gitmodules +++ b/.gitmodules @@ -23,3 +23,6 @@ path = sorc/gsi_utils.fd url = https://github.com/HenryWinterbottom-NOAA/GSI-utils branch = feature/gwdev_issue_2329 +[submodule "sorc/sorc/gsi_enkf.fd"] + path = sorc/sorc/gsi_enkf.fd + url = https://github.com/HenryWinterbottom-NOAA/GSI diff --git a/sorc/sorc/gsi_enkf.fd b/sorc/sorc/gsi_enkf.fd new file mode 160000 index 0000000000..fca6bea4e5 --- /dev/null +++ b/sorc/sorc/gsi_enkf.fd @@ -0,0 +1 @@ +Subproject commit fca6bea4e5c737efe53fd053ca9cc69033f847f7 From 7c20ea0139997dd9399174f222c74c9e6f541f5c Mon Sep 17 00:00:00 2001 From: HenryWinterbottom-NOAA Date: Wed, 6 Mar 2024 15:44:31 +0000 Subject: [PATCH 16/63] Updates for GSI ENKF. --- .gitmodules | 4 ++++ sorc/gsi_enkf.fd | 1 + 2 files changed, 5 insertions(+) create mode 160000 sorc/gsi_enkf.fd diff --git a/.gitmodules b/.gitmodules index 3fa8b9cd58..daeb70d6f2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -26,3 +26,7 @@ [submodule "sorc/sorc/gsi_enkf.fd"] path = sorc/sorc/gsi_enkf.fd url = https://github.com/HenryWinterbottom-NOAA/GSI +[submodule "sorc/gsi_enkf.fd"] + path = sorc/gsi_enkf.fd + url = https://github.com/HenryWinterbottom-NOAA/GSI + branch = feature/gwdev_issue_2329 diff --git a/sorc/gsi_enkf.fd b/sorc/gsi_enkf.fd new file mode 160000 index 0000000000..f015cc4414 --- /dev/null +++ b/sorc/gsi_enkf.fd @@ -0,0 +1 @@ +Subproject commit f015cc441490684a4e47801b2b458893ef580477 From 9e6efafb885208d33265752515804b4956bc8f78 Mon Sep 17 00:00:00 2001 From: HenryWinterbottom-NOAA Date: Wed, 6 Mar 2024 15:53:54 +0000 Subject: [PATCH 17/63] Removed hanging directory. --- sorc/sorc/gsi_enkf.fd | 1 - 1 file changed, 1 deletion(-) delete mode 160000 sorc/sorc/gsi_enkf.fd diff --git a/sorc/sorc/gsi_enkf.fd b/sorc/sorc/gsi_enkf.fd deleted file mode 160000 index fca6bea4e5..0000000000 --- a/sorc/sorc/gsi_enkf.fd +++ /dev/null @@ -1 +0,0 @@ -Subproject commit fca6bea4e5c737efe53fd053ca9cc69033f847f7 From 7553e84445865e340f4c36960d1791f0ba83f17a Mon Sep 17 00:00:00 2001 From: HenryWinterbottom-NOAA Date: Wed, 6 Mar 2024 18:54:59 +0000 Subject: [PATCH 18/63] Removed submodule. --- .gitmodules | 4 ---- sorc/ufs_model.fd | 1 - 2 files changed, 5 deletions(-) delete mode 160000 sorc/ufs_model.fd diff --git a/.gitmodules b/.gitmodules index 68320976e7..4b8c31e5bc 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,3 @@ -[submodule "sorc/ufs_model.fd"] - path = sorc/ufs_model.fd - url = https://github.com/ufs-community/ufs-weather-model - ignore = dirty [submodule "sorc/wxflow"] path = sorc/wxflow url = https://github.com/NOAA-EMC/wxflow diff --git a/sorc/ufs_model.fd b/sorc/ufs_model.fd deleted file mode 160000 index 6988662728..0000000000 --- a/sorc/ufs_model.fd +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 698866272846e8c0f8f61ddb1b20d6463460cd63 From 25e2483b8c7649bab8b3e3599a7487362cbd0d62 Mon Sep 17 00:00:00 2001 From: HenryWinterbottom-NOAA Date: Wed, 6 Mar 2024 19:06:01 +0000 Subject: [PATCH 19/63] Rocky-8 updates. --- .gitmodules | 4 ++++ modulefiles/module_gwsetup.hera.lua | 1 - sorc/ufs_model.fd | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) create mode 160000 sorc/ufs_model.fd diff --git a/.gitmodules b/.gitmodules index 4b8c31e5bc..dcd36c6ab6 100644 --- a/.gitmodules +++ b/.gitmodules @@ -26,3 +26,7 @@ [submodule "sorc/gdas.cd"] path = sorc/gdas.cd url = https://github.com/NOAA-EMC/GDASApp.git +[submodule "sorc/ufs_model.fd"] + path = sorc/ufs_model.fd + url = https://github.com/HenryWinterbottom-NOAA/ufs-weather-model + branch = feature/gwdev_issue_2329 diff --git a/modulefiles/module_gwsetup.hera.lua b/modulefiles/module_gwsetup.hera.lua index 31cb1d9f4f..696f9577be 100644 --- a/modulefiles/module_gwsetup.hera.lua +++ b/modulefiles/module_gwsetup.hera.lua @@ -4,7 +4,6 @@ Load environment to run GFS workflow setup scripts on Hera load(pathJoin("rocoto")) ---prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev/install/modulefiles/Core") prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev-rocky8/install/modulefiles/Core") local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" diff --git a/sorc/ufs_model.fd b/sorc/ufs_model.fd new file mode 160000 index 0000000000..5262eae62d --- /dev/null +++ b/sorc/ufs_model.fd @@ -0,0 +1 @@ +Subproject commit 5262eae62d325813420b4d5be653936cf226d457 From f21738b91cf533b5a13192a16e3102fe313e5ee4 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Thu, 7 Mar 2024 14:06:50 -0500 Subject: [PATCH 20/63] fix shellnorm errors --- parm/config/gefs/config.resources | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources index afcb6f1ed6..0be7e864a1 100644 --- a/parm/config/gefs/config.resources +++ b/parm/config/gefs/config.resources @@ -191,11 +191,11 @@ case ${step} in if [[ "${_CDUMP}" =~ "gfs" ]]; then declare -x "npe_${step}_gfs"="${NTASKS_TOT}" - declare -x "nth_${step}_gfs"=${UFS_THREADS} + declare -x "nth_${step}_gfs"="${UFS_THREADS}" declare -x "npe_node_${step}_gfs"="${npe_node_max}" else declare -x "npe_${step}"="${NTASKS_TOT}" - declare -x "nth_${step}"=${UFS_THREADS} + declare -x "nth_${step}"="${UFS_THREADS}" declare -x "npe_node_${step}"="${npe_node_max}" fi From 33d3eccf0687f6ef46f23e6f9e68751ff58d50d6 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Thu, 7 Mar 2024 14:20:26 -0500 Subject: [PATCH 21/63] update config.resources in the repo with the one used in test --- parm/config/gfs/config.resources | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index cf5665fa19..f63690a529 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -631,11 +631,11 @@ case ${step} in if [[ "${_CDUMP}" =~ "gfs" ]]; then declare -x "npe_${step}_gfs"="${NTASKS_TOT}" - declare -x "nth_${step}_gfs"=1 # ESMF handles threading for the UFS-weather-model + declare -x "nth_${step}_gfs"="${UFS_THREADS}" declare -x "npe_node_${step}_gfs"="${npe_node_max}" else declare -x "npe_${step}"="${NTASKS_TOT}" - declare -x "nth_${step}"=1 # ESMF handles threading for the UFS-weather-model + declare -x "nth_${step}"="${UFS_THREADS}" declare -x "npe_node_${step}"="${npe_node_max}" fi From f6dc43064d4fff43a2678ffc6ebff10344707225 Mon Sep 17 00:00:00 2001 From: HenryWinterbottom-NOAA Date: Tue, 12 Mar 2024 20:17:11 +0000 Subject: [PATCH 22/63] Turned off GSI verification for RDHPCS Hera. --- parm/config/gfs/config.base | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 48a9c728a4..e705bae368 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -380,8 +380,8 @@ export FITSARC="YES" export FHMAX_FITS=132 [[ "${FHMAX_FITS}" -gt "${FHMAX_GFS}" ]] && export FHMAX_FITS=${FHMAX_GFS} -# The monitor jobs are not yet supported for JEDIATMVAR -if [[ ${DO_JEDIATMVAR} = "YES" ]]; then +# The monitor jobs are not yet supported for JEDIATMVAR or RDHPCS Hera for Rocky-8. +if [[ ${DO_JEDIATMVAR} = "YES" ]] || [[ ${machine} = "HERA" ]]; then export DO_VERFOZN="NO" # Ozone data assimilation monitoring export DO_VERFRAD="NO" # Radiance data assimilation monitoring export DO_VMINMON="NO" # GSI minimization monitoring From e7e2b9a3d760537073221efbfc547682938a9058 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Wed, 13 Mar 2024 10:16:48 -0400 Subject: [PATCH 23/63] replace HOMEgfs/parm with PARMgfs to be consistent w/ gw --- parm/config/gefs/config.ufs | 14 +++++++------- parm/config/gfs/config.ufs | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/parm/config/gefs/config.ufs b/parm/config/gefs/config.ufs index 6e423865ee..b8695b6dbb 100644 --- a/parm/config/gefs/config.ufs +++ b/parm/config/gefs/config.ufs @@ -452,25 +452,25 @@ if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then fi case "${model_list}" in atm) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.atm${tmpl_suffix:-}.IN" + default_template="${PARMgfs}/ufs/ufs.configure.atm${tmpl_suffix:-}.IN" ;; atm.aero) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.atmaero${tmpl_suffix:-}.IN" + default_template="${PARMgfs}/ufs/ufs.configure.atmaero${tmpl_suffix:-}.IN" ;; atm.wave) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.leapfrog_atm_wav${tmpl_suffix:-}.IN" + default_template="${PARMgfs}/ufs/ufs.configure.leapfrog_atm_wav${tmpl_suffix:-}.IN" ;; atm.ocean.ice) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2s${tmpl_suffix:-}.IN" + default_template="${PARMgfs}/ufs/ufs.configure.s2s${tmpl_suffix:-}.IN" ;; atm.ocean.ice.aero) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2sa${tmpl_suffix:-}.IN" + default_template="${PARMgfs}/ufs/ufs.configure.s2sa${tmpl_suffix:-}.IN" ;; atm.ocean.ice.wave) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2sw${tmpl_suffix:-}.IN" + default_template="${PARMgfs}/ufs/ufs.configure.s2sw${tmpl_suffix:-}.IN" ;; atm.ocean.ice.wave.aero) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2swa${tmpl_suffix:-}.IN" + default_template="${PARMgfs}/ufs/ufs.configure.s2swa${tmpl_suffix:-}.IN" ;; *) echo "FATAL ERROR: Unsupported UFSWM configuration for ${model_list}" diff --git a/parm/config/gfs/config.ufs b/parm/config/gfs/config.ufs index 15366ab519..0db6f090a5 100644 --- a/parm/config/gfs/config.ufs +++ b/parm/config/gfs/config.ufs @@ -460,25 +460,25 @@ if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then fi case "${model_list}" in atm) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.atm${tmpl_suffix:-}.IN" + default_template="${PARMgfs}/ufs/ufs.configure.atm${tmpl_suffix:-}.IN" ;; atm.aero) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.atmaero${tmpl_suffix:-}.IN" + default_template="${PARMgfs}/ufs/ufs.configure.atmaero${tmpl_suffix:-}.IN" ;; atm.wave) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.leapfrog_atm_wav${tmpl_suffix:-}.IN" + default_template="${PARMgfs}/ufs/ufs.configure.leapfrog_atm_wav${tmpl_suffix:-}.IN" ;; atm.ocean.ice) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2s${tmpl_suffix:-}.IN" + default_template="${PARMgfs}/ufs/ufs.configure.s2s${tmpl_suffix:-}.IN" ;; atm.ocean.ice.aero) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2sa${tmpl_suffix:-}.IN" + default_template="${PARMgfs}/ufs/ufs.configure.s2sa${tmpl_suffix:-}.IN" ;; atm.ocean.ice.wave) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2sw${tmpl_suffix:-}.IN" + default_template="${PARMgfs}/ufs/ufs.configure.s2sw${tmpl_suffix:-}.IN" ;; atm.ocean.ice.wave.aero) - default_template="${HOMEgfs}/parm/ufs/ufs.configure.s2swa${tmpl_suffix:-}.IN" + default_template="${PARMgfs}/ufs/ufs.configure.s2swa${tmpl_suffix:-}.IN" ;; *) echo "FATAL ERROR: Unsupported UFSWM configuration for ${model_list}" From e60ae211a7daf93188127a039f4ec99648d8fc68 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Wed, 13 Mar 2024 10:18:34 -0400 Subject: [PATCH 24/63] fix mixed-indent in .gitmodules --- .gitmodules | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitmodules b/.gitmodules index 0c89ebad72..cb2759bb86 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,9 @@ [submodule "sorc/ufs_model.fd"] - path = sorc/ufs_model.fd - #url = https://github.com/ufs-community/ufs-weather-model - url = https://github.com/aerorahul/ufs-weather-model + path = sorc/ufs_model.fd + #url = https://github.com/ufs-community/ufs-weather-model + url = https://github.com/aerorahul/ufs-weather-model branch = feature/ufs-threading - ignore = dirty + ignore = dirty [submodule "sorc/wxflow"] path = sorc/wxflow url = https://github.com/NOAA-EMC/wxflow From 33a3553cdc5f83c72017e4665ab471b14313032b Mon Sep 17 00:00:00 2001 From: HenryWinterbottom-NOAA Date: Wed, 13 Mar 2024 15:30:15 +0000 Subject: [PATCH 25/63] Trying again to update hashes. --- sorc/ufs_model.fd | 2 +- sorc/ufs_utils.fd | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sorc/ufs_model.fd b/sorc/ufs_model.fd index 5262eae62d..d5bc038bd1 160000 --- a/sorc/ufs_model.fd +++ b/sorc/ufs_model.fd @@ -1 +1 @@ -Subproject commit 5262eae62d325813420b4d5be653936cf226d457 +Subproject commit d5bc038bd10e6513dfba825923b9e109f7083245 diff --git a/sorc/ufs_utils.fd b/sorc/ufs_utils.fd index 580f5ba80d..5912ca3dc1 160000 --- a/sorc/ufs_utils.fd +++ b/sorc/ufs_utils.fd @@ -1 +1 @@ -Subproject commit 580f5ba80d78666321ba3cc08485f43cace8e655 +Subproject commit 5912ca3dc1dbec2f96d64a1995d5193c967cb24e From 18667e387f88f8ab466984a27d0630e456fb2115 Mon Sep 17 00:00:00 2001 From: HenryWinterbottom-NOAA Date: Wed, 13 Mar 2024 19:29:40 +0000 Subject: [PATCH 26/63] Update UPP hash to Rocky8 spack-stack --- sorc/ufs_model.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/ufs_model.fd b/sorc/ufs_model.fd index d5bc038bd1..2f693e781c 160000 --- a/sorc/ufs_model.fd +++ b/sorc/ufs_model.fd @@ -1 +1 @@ -Subproject commit d5bc038bd10e6513dfba825923b9e109f7083245 +Subproject commit 2f693e781c4f0b58712029b5b07256977b945f4f From cd90a3441325f9ef1acb671d0c568ce2a0ce4736 Mon Sep 17 00:00:00 2001 From: HenryWinterbottom-NOAA Date: Wed, 13 Mar 2024 20:26:01 +0000 Subject: [PATCH 27/63] Rocky-8 updates. --- sorc/ufs_model.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/ufs_model.fd b/sorc/ufs_model.fd index 2f693e781c..b661f362e5 160000 --- a/sorc/ufs_model.fd +++ b/sorc/ufs_model.fd @@ -1 +1 @@ -Subproject commit 2f693e781c4f0b58712029b5b07256977b945f4f +Subproject commit b661f362e52eb45e0312b62fa8924b275be76b5b From 8ed5ad3313d0de16fe433ce8884dd135fbc7f1a6 Mon Sep 17 00:00:00 2001 From: HenryWinterbottom-NOAA Date: Thu, 14 Mar 2024 22:42:51 +0000 Subject: [PATCH 28/63] Removed submodule. --- .gitmodules | 3 --- sorc/gsi_monitor.fd | 1 - 2 files changed, 4 deletions(-) delete mode 160000 sorc/gsi_monitor.fd diff --git a/.gitmodules b/.gitmodules index dcd36c6ab6..f4a85f1206 100644 --- a/.gitmodules +++ b/.gitmodules @@ -5,9 +5,6 @@ path = sorc/gfs_utils.fd url = https://github.com/HenryWinterbottom-NOAA/gfs-utils branch = feature/gwdev_issue_2329 -[submodule "sorc/gsi_monitor.fd"] - path = sorc/gsi_monitor.fd - url = https://github.com/NOAA-EMC/GSI-Monitor.git [submodule "sorc/ufs_utils.fd"] path = sorc/ufs_utils.fd url = https://github.com/HenryWinterbottom-NOAA/UFS_UTILS diff --git a/sorc/gsi_monitor.fd b/sorc/gsi_monitor.fd deleted file mode 160000 index 149cd811f2..0000000000 --- a/sorc/gsi_monitor.fd +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 149cd811f273883e7d6400eded7b980acd65ca2b From 95b234639a8bd8df9b54f3862b66976f8d33cbb9 Mon Sep 17 00:00:00 2001 From: HenryWinterbottom-NOAA Date: Thu, 14 Mar 2024 22:45:19 +0000 Subject: [PATCH 29/63] Updated hash for gsi_monitor.fd. --- .gitmodules | 4 ++++ sorc/gsi_monitor.fd | 1 + 2 files changed, 5 insertions(+) create mode 160000 sorc/gsi_monitor.fd diff --git a/.gitmodules b/.gitmodules index f4a85f1206..1ea57b1f00 100644 --- a/.gitmodules +++ b/.gitmodules @@ -27,3 +27,7 @@ path = sorc/ufs_model.fd url = https://github.com/HenryWinterbottom-NOAA/ufs-weather-model branch = feature/gwdev_issue_2329 +[submodule "sorc/gsi_monitor.fd"] + path = sorc/gsi_monitor.fd + url = https://github.com/HenryWinterbottom-NOAA/GSI-Monitor + branch = feature/gwdev_issue_2329 diff --git a/sorc/gsi_monitor.fd b/sorc/gsi_monitor.fd new file mode 160000 index 0000000000..610300ded0 --- /dev/null +++ b/sorc/gsi_monitor.fd @@ -0,0 +1 @@ +Subproject commit 610300ded0dc46a411b4b90d53d585adc0944d94 From 510ad81d27cebed0779fc92f22a6ef82b5ac6650 Mon Sep 17 00:00:00 2001 From: HenryWinterbottom-NOAA Date: Tue, 19 Mar 2024 18:26:30 +0000 Subject: [PATCH 30/63] UFSWM updates. --- sorc/ufs_model.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/ufs_model.fd b/sorc/ufs_model.fd index b661f362e5..d7d9976c60 160000 --- a/sorc/ufs_model.fd +++ b/sorc/ufs_model.fd @@ -1 +1 @@ -Subproject commit b661f362e52eb45e0312b62fa8924b275be76b5b +Subproject commit d7d9976c60fcee011597897fe4ac7faf51bec833 From c3cb6115f7f9c2106b59c868db908d7bc5c86625 Mon Sep 17 00:00:00 2001 From: HenryWinterbottom-NOAA Date: Tue, 19 Mar 2024 19:03:57 +0000 Subject: [PATCH 31/63] Updates for prepobs and Fit2Obs. --- modulefiles/module_base.hera.lua | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/modulefiles/module_base.hera.lua b/modulefiles/module_base.hera.lua index 8703100e76..7510d7f6cd 100644 --- a/modulefiles/module_base.hera.lua +++ b/modulefiles/module_base.hera.lua @@ -41,11 +41,10 @@ load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None"))) setenv("WGRIB2","wgrib2") setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) ---prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) -prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/feature-GFSv17_com_reorg_log_update/modulefiles")) +prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/feature-hera-rocky8/modulefiles")) load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None"))) -prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) +prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/Fit2Obs/feature-hera-rocky8/modulefiles")) load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None"))) whatis("Description: GFS run environment") From 8d4243f1d58def4a026501a4bd8fd44b558ef4da Mon Sep 17 00:00:00 2001 From: HenryWinterbottom-NOAA Date: Wed, 20 Mar 2024 17:48:39 +0000 Subject: [PATCH 32/63] Removed broken submodule. --- .gitmodules | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index 1ea57b1f00..5f6703f9be 100644 --- a/.gitmodules +++ b/.gitmodules @@ -17,9 +17,6 @@ path = sorc/gsi_enkf.fd url = https://github.com/HenryWinterbottom-NOAA/GSI branch = feature/gwdev_issue_2329 -[submodule "sorc/verif-global.fd"] - path = sorc/verif-global.fd - url = https://github.com/NOAA-EMC/EMC_verif-global.git [submodule "sorc/gdas.cd"] path = sorc/gdas.cd url = https://github.com/NOAA-EMC/GDASApp.git From 4dec833e0a3b29aa04f95097e2f615d8cae10561 Mon Sep 17 00:00:00 2001 From: HenryWinterbottom-NOAA Date: Wed, 20 Mar 2024 17:49:27 +0000 Subject: [PATCH 33/63] Adding submodule sorc/verif-global.fd. --- .gitmodules | 3 +++ sorc/verif-global.fd | 1 + 2 files changed, 4 insertions(+) create mode 160000 sorc/verif-global.fd diff --git a/.gitmodules b/.gitmodules index 5f6703f9be..847d1955f7 100644 --- a/.gitmodules +++ b/.gitmodules @@ -28,3 +28,6 @@ path = sorc/gsi_monitor.fd url = https://github.com/HenryWinterbottom-NOAA/GSI-Monitor branch = feature/gwdev_issue_2329 +[submodule "sorc/verif-global.fd"] + path = sorc/verif-global.fd + url = https://github.com/NOAA-EMC/EMC_verif-global.git diff --git a/sorc/verif-global.fd b/sorc/verif-global.fd new file mode 160000 index 0000000000..bd1c8f62a1 --- /dev/null +++ b/sorc/verif-global.fd @@ -0,0 +1 @@ +Subproject commit bd1c8f62a1878051e34ff7c6f6a4dd290381f1ef From bd57aadec9bbad283f9ef26ea554917425741ccc Mon Sep 17 00:00:00 2001 From: HenryWinterbottom-NOAA Date: Wed, 20 Mar 2024 18:31:49 +0000 Subject: [PATCH 34/63] Updated to Rocky-8 stack. --- versions/build.hera.ver | 2 +- versions/run.hera.ver | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/versions/build.hera.ver b/versions/build.hera.ver index 263e967a97..337d5c32da 100644 --- a/versions/build.hera.ver +++ b/versions/build.hera.ver @@ -1,5 +1,5 @@ export stack_intel_ver=2021.5.0 export stack_impi_ver=2021.5.1 -export spack_env=gsi-addon-dev +export spack_env=gsi-addon-dev-rocky8 source "${HOMEgfs:-}/versions/build.spack.ver" export spack_mod_path="/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core" diff --git a/versions/run.hera.ver b/versions/run.hera.ver index 4529d34821..b358f9d495 100644 --- a/versions/run.hera.ver +++ b/versions/run.hera.ver @@ -1,6 +1,6 @@ export stack_intel_ver=2021.5.0 export stack_impi_ver=2021.5.1 -export spack_env=gsi-addon-dev +export spack_env=gsi-addon-dev-rocky8 export hpss_ver=hpss export ncl_ver=6.6.2 From 613418b088a687e06422c618466402dd20dfcb0a Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Wed, 20 Mar 2024 04:07:49 -0500 Subject: [PATCH 35/63] Modify APP based on RUN There is a need to change which coupled components are on depending on the current `RUN`. To facilitate this, the `APP` is modified prior to the setting of the `DO_` variables based on `RUN`, turning off components as desired. This new system also replaces the `DO__ENKF` switches that were formerly used to turn components off for the ensemble. Resolves #2318 --- parm/config/gfs/config.base | 13 +++++++++++++ parm/config/gfs/config.efcs | 6 ------ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 169fa37430..a7ae28d4b8 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -142,6 +142,18 @@ export DBNROOT=${DBNROOT:-${UTILROOT:-}/fakedbn} # APP settings export APP=@APP@ +# Adjust APP based on RUN +case "${RUN}" in + gfs) # Turn off aerosols + APP="${APP/%A}" + ;; + enkf*) # Turn off aerosols and waves + APP="${APP/%+[WA]}" + ;; + *) # Keep app unchanged + ;; +esac + # Defaults: export DO_ATM="YES" export DO_COUPLED="NO" @@ -162,6 +174,7 @@ export CASE="@CASECTL@" export CASE_ENS="@CASEENS@" export OCNRES="@OCNRES@" export ICERES="${OCNRES}" + # These are the currently recommended grid-combinations case "${CASE}" in "C48") diff --git a/parm/config/gfs/config.efcs b/parm/config/gfs/config.efcs index effd443730..402ba64087 100644 --- a/parm/config/gfs/config.efcs +++ b/parm/config/gfs/config.efcs @@ -5,12 +5,6 @@ echo "BEGIN: config.efcs" -# Turn off components in ensemble -# export DO_AERO="NO" -# export DO_OCN="NO" -# export DO_ICE="NO" -export DO_WAVE="NO" - export CASE="${CASE_ENS}" # Source model specific information that is resolution dependent From eb9bef1ee1622f4b3649d830c8449af187d104df Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Wed, 20 Mar 2024 12:29:28 -0500 Subject: [PATCH 36/63] Correct APP substitution for enkf --- parm/config/gfs/config.base | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index a7ae28d4b8..234debcefb 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -148,7 +148,7 @@ case "${RUN}" in APP="${APP/%A}" ;; enkf*) # Turn off aerosols and waves - APP="${APP/%+[WA]}" + APP="${APP/%+([WA])}" ;; *) # Keep app unchanged ;; From e5dd245785e35503d5963352824138491565d800 Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Wed, 20 Mar 2024 12:57:51 -0500 Subject: [PATCH 37/63] Turn on extglob for APP adjustment --- parm/config/gfs/config.base | 2 ++ 1 file changed, 2 insertions(+) diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 234debcefb..e9ece39404 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -142,6 +142,7 @@ export DBNROOT=${DBNROOT:-${UTILROOT:-}/fakedbn} # APP settings export APP=@APP@ +shopt -s extglob # Adjust APP based on RUN case "${RUN}" in gfs) # Turn off aerosols @@ -153,6 +154,7 @@ case "${RUN}" in *) # Keep app unchanged ;; esac +shopt -u extglob # Defaults: export DO_ATM="YES" From 6aedab214e3e8617e4fd3143f8f35b5fcaff3f4e Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Wed, 20 Mar 2024 14:57:25 -0500 Subject: [PATCH 38/63] Add S2SWA as valid app option --- workflow/setup_expt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflow/setup_expt.py b/workflow/setup_expt.py index 90e890910f..c689ec2ae7 100755 --- a/workflow/setup_expt.py +++ b/workflow/setup_expt.py @@ -385,7 +385,7 @@ def input_args(*argv): Method to collect user arguments for `setup_expt.py` """ - ufs_apps = ['ATM', 'ATMA', 'ATMW', 'S2S', 'S2SA', 'S2SW'] + ufs_apps = ['ATM', 'ATMA', 'ATMW', 'S2S', 'S2SA', 'S2SW', 'S2SWA'] def _common_args(parser): parser.add_argument('--pslot', help='parallel experiment name', @@ -433,7 +433,7 @@ def _gfs_or_gefs_ensemble_args(parser): def _gfs_or_gefs_forecast_args(parser): parser.add_argument('--app', help='UFS application', type=str, - choices=ufs_apps + ['S2SWA'], required=False, default='ATM') + choices=ufs_apps, required=False, default='ATM') parser.add_argument('--gfs_cyc', help='Number of forecasts per day', type=int, choices=[1, 2, 4], default=1, required=False) return parser From 65830b8b81f405cec8372540857b05dc03fce2c6 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Thu, 21 Mar 2024 09:44:44 -0400 Subject: [PATCH 39/63] update ufs-weather-model hash to 126db7a5 and in the authoritative repo --- .gitmodules | 4 +--- sorc/ufs_model.fd | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.gitmodules b/.gitmodules index cb2759bb86..5c9e569243 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,8 +1,6 @@ [submodule "sorc/ufs_model.fd"] path = sorc/ufs_model.fd - #url = https://github.com/ufs-community/ufs-weather-model - url = https://github.com/aerorahul/ufs-weather-model - branch = feature/ufs-threading + url = https://github.com/ufs-community/ufs-weather-model ignore = dirty [submodule "sorc/wxflow"] path = sorc/wxflow diff --git a/sorc/ufs_model.fd b/sorc/ufs_model.fd index a07bd60c07..126db7a514 160000 --- a/sorc/ufs_model.fd +++ b/sorc/ufs_model.fd @@ -1 +1 @@ -Subproject commit a07bd60c07be7222d0eea90e7459376304fbcafe +Subproject commit 126db7a5145319cb6a8b3d272215efcd5b88562f From c558151e24bd21b19816c600cd2dfc561838c0ec Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Thu, 21 Mar 2024 10:00:42 -0400 Subject: [PATCH 40/63] add a newline at the end of diag_table_aod --- parm/ufs/fv3/diag_table_aod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/ufs/fv3/diag_table_aod b/parm/ufs/fv3/diag_table_aod index 0de51b66d8..fd8aee1791 100644 --- a/parm/ufs/fv3/diag_table_aod +++ b/parm/ufs/fv3/diag_table_aod @@ -3,4 +3,4 @@ "gfs_phys", "SU_AOD_550", "su_aod550", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "BC_AOD_550", "bc_aod550", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "OC_AOD_550", "oc_aod550", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "SS_AOD_550", "ss_aod550", "fv3_history2d", "all", .false., "none", 2 \ No newline at end of file +"gfs_phys", "SS_AOD_550", "ss_aod550", "fv3_history2d", "all", .false., "none", 2 From 094c7da9a038e750901d762e67a8aa4ed6288d10 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Thu, 21 Mar 2024 10:16:54 -0400 Subject: [PATCH 41/63] increase memory for offline upp on wcoss2 for C768 --- parm/config/gfs/config.resources | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index f2e32d8b0f..9a15a9ad4a 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -706,9 +706,13 @@ case ${step} in "C48" | "C96") export npe_upp=${CASE:1} ;; - "C192" | "C384" | "C768") + "C192" | "C384") export npe_upp=120 - export memory_upp="48GB" + export memory_upp="96GB" + ;; + "C768") + export npe_upp=120 + if [[ ${machine} == "WCOSS2" ]]; then export memory_upp="480GB" ; fi ;; *) echo "FATAL ERROR: Resources not defined for job ${job} at resolution ${CASE}" From 9d8060e2fb369293a3d5b40ef25424f846d7b397 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Thu, 21 Mar 2024 10:20:06 -0400 Subject: [PATCH 42/63] Update sorc/link_workflow.sh Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> --- sorc/link_workflow.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 661c42ad12..9184e08432 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -177,7 +177,8 @@ declare -a ufs_templates=("model_configure.IN" \ "ufs.configure.s2sw_esmf.IN" \ "ufs.configure.s2swa.IN" \ "ufs.configure.s2swa_esmf.IN" \ - "ufs.configure.leapfrog_atm_wav.IN" ) + "ufs.configure.leapfrog_atm_wav.IN" \ + "ufs.configure.leapfrog_atm_wav_esmf.IN" ) for file in "${ufs_templates[@]}"; do [[ -s "${file}" ]] && rm -f "${file}" ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/tests/parm/${file}" . From a33ae0fe0575389db407d5a6d8718ec021239b02 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 21 Mar 2024 14:51:49 +0000 Subject: [PATCH 43/63] Fix earc for which nmem isn't a multiple of 10. --- ush/hpssarch_gen.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/hpssarch_gen.sh b/ush/hpssarch_gen.sh index eadd465421..8b61cac948 100755 --- a/ush/hpssarch_gen.sh +++ b/ush/hpssarch_gen.sh @@ -681,7 +681,7 @@ if [[ ${type} == "enkfgdas" || ${type} == "enkfgfs" ]]; then touch "${DATA}/${RUN}_restartb_grp${n}.txt" m=1 - while (( m <= NMEM_EARCGRP )); do + while (( m <= NMEM_EARCGRP && (n-1)*NMEM_EARCGRP+m <= NMEM_ENS )); do nm=$(((n-1)*NMEM_EARCGRP+m)) mem=$(printf %03i ${nm}) head="${RUN}.t${cyc}z." From ddd3ea05c1066b0e6305a598068437d53fc37376 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Thu, 21 Mar 2024 10:55:05 -0400 Subject: [PATCH 44/63] Update parm/config/gfs/config.resources --- parm/config/gfs/config.resources | 1 + 1 file changed, 1 insertion(+) diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 9a15a9ad4a..98fc3b2668 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -712,6 +712,7 @@ case ${step} in ;; "C768") export npe_upp=120 + export memory_upp="96GB" if [[ ${machine} == "WCOSS2" ]]; then export memory_upp="480GB" ; fi ;; *) From db3bab4a683c87d0fbfb542b6d2595037855500d Mon Sep 17 00:00:00 2001 From: "Henry R. Winterbottom" <49202169+HenryWinterbottom-NOAA@users.noreply.github.com> Date: Thu, 21 Mar 2024 10:05:42 -0600 Subject: [PATCH 45/63] Update .gitmodules Co-authored-by: Rahul Mahajan --- .gitmodules | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index 847d1955f7..8fd0d0005f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -14,9 +14,9 @@ url = https://github.com/HenryWinterbottom-NOAA/GSI-utils branch = feature/gwdev_issue_2329 [submodule "sorc/gsi_enkf.fd"] - path = sorc/gsi_enkf.fd - url = https://github.com/HenryWinterbottom-NOAA/GSI - branch = feature/gwdev_issue_2329 + path = sorc/gsi_enkf.fd + url = https://github.com/NOAA-EMC/GSI.git + [submodule "sorc/gdas.cd"] path = sorc/gdas.cd url = https://github.com/NOAA-EMC/GDASApp.git From 695f7994b6f56e9eb588de9570f6fabb8525c8f6 Mon Sep 17 00:00:00 2001 From: "Henry R. Winterbottom" <49202169+HenryWinterbottom-NOAA@users.noreply.github.com> Date: Thu, 21 Mar 2024 10:05:49 -0600 Subject: [PATCH 46/63] Update parm/config/gfs/config.base Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> --- parm/config/gfs/config.base | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 4a026389e6..31f4825ad5 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -383,11 +383,14 @@ export FITSARC="YES" export FHMAX_FITS=132 [[ "${FHMAX_FITS}" -gt "${FHMAX_GFS}" ]] && export FHMAX_FITS=${FHMAX_GFS} -# The monitor jobs are not yet supported for JEDIATMVAR or RDHPCS Hera for Rocky-8. -if [[ ${DO_JEDIATMVAR} = "YES" ]] || [[ ${machine} = "HERA" ]]; then +# The monitor jobs are not yet supported for JEDIATMVAR. +if [[ ${DO_JEDIATMVAR} = "YES" ]]; then export DO_VERFOZN="NO" # Ozone data assimilation monitoring export DO_VERFRAD="NO" # Radiance data assimilation monitoring export DO_VMINMON="NO" # GSI minimization monitoring +# The minimization monitor is not yet supported on RDHPCS Hera for Rocky-8 +elif [[ ${machine} = "HERA" ]]; then + export DO_VMINMON="NO" # GSI minimization monitoring fi echo "END: config.base" From 668ede009b08644adaa76db4616c984f996e7afc Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Thu, 21 Mar 2024 12:13:13 -0400 Subject: [PATCH 47/63] replace reset w/ purge for orion --- ush/module-setup.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ush/module-setup.sh b/ush/module-setup.sh index 008ef7ee97..b66e3622d0 100755 --- a/ush/module-setup.sh +++ b/ush/module-setup.sh @@ -36,10 +36,10 @@ elif [[ ${MACHINE_ID} = orion* ]] ; then if ( ! eval module help > /dev/null 2>&1 ) ; then source /apps/lmod/lmod/init/bash fi - export LMOD_SYSTEM_DEFAULT_MODULES=contrib - set +u - module reset - set -u + #export LMOD_SYSTEM_DEFAULT_MODULES=git/2.28.0 # contrib has a lot of stuff we shouldn't put in MODULEPATH + #set +u + module purge # reset causes issues on Orion sometimes. + #set -u elif [[ ${MACHINE_ID} = s4* ]] ; then # We are on SSEC Wisconsin S4 From 36e667b894873b41c1d6e95469fc47753e189481 Mon Sep 17 00:00:00 2001 From: HenryWinterbottom-NOAA Date: Fri, 22 Mar 2024 14:05:01 +0000 Subject: [PATCH 48/63] Removed submodule fork paths. --- .gitmodules | 21 --------------------- sorc/gfs_utils.fd | 1 - sorc/gsi_monitor.fd | 1 - sorc/gsi_utils.fd | 1 - sorc/ufs_model.fd | 1 - sorc/ufs_utils.fd | 1 - 6 files changed, 26 deletions(-) delete mode 160000 sorc/gfs_utils.fd delete mode 160000 sorc/gsi_monitor.fd delete mode 160000 sorc/gsi_utils.fd delete mode 160000 sorc/ufs_model.fd delete mode 160000 sorc/ufs_utils.fd diff --git a/.gitmodules b/.gitmodules index 8fd0d0005f..baf4dda592 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,33 +1,12 @@ [submodule "sorc/wxflow"] path = sorc/wxflow url = https://github.com/NOAA-EMC/wxflow -[submodule "sorc/gfs_utils.fd"] - path = sorc/gfs_utils.fd - url = https://github.com/HenryWinterbottom-NOAA/gfs-utils - branch = feature/gwdev_issue_2329 -[submodule "sorc/ufs_utils.fd"] - path = sorc/ufs_utils.fd - url = https://github.com/HenryWinterbottom-NOAA/UFS_UTILS - branch = feature/gwdev_issue_2329 -[submodule "sorc/gsi_utils.fd"] - path = sorc/gsi_utils.fd - url = https://github.com/HenryWinterbottom-NOAA/GSI-utils - branch = feature/gwdev_issue_2329 [submodule "sorc/gsi_enkf.fd"] path = sorc/gsi_enkf.fd url = https://github.com/NOAA-EMC/GSI.git - [submodule "sorc/gdas.cd"] path = sorc/gdas.cd url = https://github.com/NOAA-EMC/GDASApp.git -[submodule "sorc/ufs_model.fd"] - path = sorc/ufs_model.fd - url = https://github.com/HenryWinterbottom-NOAA/ufs-weather-model - branch = feature/gwdev_issue_2329 -[submodule "sorc/gsi_monitor.fd"] - path = sorc/gsi_monitor.fd - url = https://github.com/HenryWinterbottom-NOAA/GSI-Monitor - branch = feature/gwdev_issue_2329 [submodule "sorc/verif-global.fd"] path = sorc/verif-global.fd url = https://github.com/NOAA-EMC/EMC_verif-global.git diff --git a/sorc/gfs_utils.fd b/sorc/gfs_utils.fd deleted file mode 160000 index 4b7f6095d2..0000000000 --- a/sorc/gfs_utils.fd +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 4b7f6095d260b7fcd9c99c337454e170f1aa7f2f diff --git a/sorc/gsi_monitor.fd b/sorc/gsi_monitor.fd deleted file mode 160000 index 610300ded0..0000000000 --- a/sorc/gsi_monitor.fd +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 610300ded0dc46a411b4b90d53d585adc0944d94 diff --git a/sorc/gsi_utils.fd b/sorc/gsi_utils.fd deleted file mode 160000 index c52f065e58..0000000000 --- a/sorc/gsi_utils.fd +++ /dev/null @@ -1 +0,0 @@ -Subproject commit c52f065e587951d6f6b024695983eb3a7769d6ea diff --git a/sorc/ufs_model.fd b/sorc/ufs_model.fd deleted file mode 160000 index d7d9976c60..0000000000 --- a/sorc/ufs_model.fd +++ /dev/null @@ -1 +0,0 @@ -Subproject commit d7d9976c60fcee011597897fe4ac7faf51bec833 diff --git a/sorc/ufs_utils.fd b/sorc/ufs_utils.fd deleted file mode 160000 index 5912ca3dc1..0000000000 --- a/sorc/ufs_utils.fd +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 5912ca3dc1dbec2f96d64a1995d5193c967cb24e From bd3201c5999113bb05fe9f62427a069fb6d5fcea Mon Sep 17 00:00:00 2001 From: HenryWinterbottom-NOAA Date: Fri, 22 Mar 2024 14:15:40 +0000 Subject: [PATCH 49/63] Removed incorrect submodule path. --- .gitmodules | 12 ++++++++++++ sorc/gfs_utils.fd | 1 + sorc/gsi_monitor.fd | 1 + sorc/ufs_model.fd | 1 + sorc/ufs_utils.fd | 1 + 5 files changed, 16 insertions(+) create mode 160000 sorc/gfs_utils.fd create mode 160000 sorc/gsi_monitor.fd create mode 160000 sorc/ufs_model.fd create mode 160000 sorc/ufs_utils.fd diff --git a/.gitmodules b/.gitmodules index baf4dda592..77e86dcfa0 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,3 +10,15 @@ [submodule "sorc/verif-global.fd"] path = sorc/verif-global.fd url = https://github.com/NOAA-EMC/EMC_verif-global.git +[submodule "sorc/gsi_monitor.fd"] + path = sorc/gsi_monitor.fd + url = https://github.com/NOAA-EMC/GSI-Monitor +[submodule "sorc/gfs_utils.fd"] + path = sorc/gfs_utils.fd + url = https://github.com/NOAA-EMC/gfs-utils +[submodule "sorc/ufs_utils.fd"] + path = sorc/ufs_utils.fd + url = https://github.com/ufs-community/UFS_UTILS +[submodule "sorc/ufs_model.fd"] + path = sorc/ufs_model.fd + url = https://github.com/ufs-community/ufs-weather-model diff --git a/sorc/gfs_utils.fd b/sorc/gfs_utils.fd new file mode 160000 index 0000000000..de3708bfb0 --- /dev/null +++ b/sorc/gfs_utils.fd @@ -0,0 +1 @@ +Subproject commit de3708bfb00cd51900e813b84fdf2a3be5d398b0 diff --git a/sorc/gsi_monitor.fd b/sorc/gsi_monitor.fd new file mode 160000 index 0000000000..3ecfe07913 --- /dev/null +++ b/sorc/gsi_monitor.fd @@ -0,0 +1 @@ +Subproject commit 3ecfe0791391cee11cb0e3a3bcc16957b37057e0 diff --git a/sorc/ufs_model.fd b/sorc/ufs_model.fd new file mode 160000 index 0000000000..7fdb58cad0 --- /dev/null +++ b/sorc/ufs_model.fd @@ -0,0 +1 @@ +Subproject commit 7fdb58cad0dad2f62ce7813c6719554d1c5a17af diff --git a/sorc/ufs_utils.fd b/sorc/ufs_utils.fd new file mode 160000 index 0000000000..f42fae239d --- /dev/null +++ b/sorc/ufs_utils.fd @@ -0,0 +1 @@ +Subproject commit f42fae239d0824f7b9a83c9afdc3d980894c7df8 From 2e0169732ffbf200962ca15f611c1593f3ba4be4 Mon Sep 17 00:00:00 2001 From: HenryWinterbottom-NOAA Date: Fri, 22 Mar 2024 14:21:56 +0000 Subject: [PATCH 50/63] Updated submodules; pointing to tip of UPP until Rocky-8 issues are resolved. --- .gitmodules | 6 ++++++ sorc/gsi_utils.fd | 1 + sorc/link_workflow.sh | 9 +++++---- sorc/upp.fd | 1 + 4 files changed, 13 insertions(+), 4 deletions(-) create mode 160000 sorc/gsi_utils.fd create mode 160000 sorc/upp.fd diff --git a/.gitmodules b/.gitmodules index 77e86dcfa0..934ef43d95 100644 --- a/.gitmodules +++ b/.gitmodules @@ -22,3 +22,9 @@ [submodule "sorc/ufs_model.fd"] path = sorc/ufs_model.fd url = https://github.com/ufs-community/ufs-weather-model +[submodule "sorc/gsi_utils.fd"] + path = sorc/gsi_utils.fd + url = https://github.com/NOAA-EMC/GSI-utils +[submodule "sorc/upp.fd"] + path = sorc/upp.fd + url = https://github.com/NOAA-EMC/UPP diff --git a/sorc/gsi_utils.fd b/sorc/gsi_utils.fd new file mode 160000 index 0000000000..67b014d8d3 --- /dev/null +++ b/sorc/gsi_utils.fd @@ -0,0 +1 @@ +Subproject commit 67b014d8d3e5acc1d21aca15e3fe2d66d327a206 diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 2b54f3ea10..ee749f4d51 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -345,10 +345,11 @@ fi #--link source code directories #------------------------------ cd "${HOMEgfs}/sorc" || exit 8 -if [[ -d ufs_model.fd ]]; then - [[ -d upp.fd ]] && rm -rf upp.fd - ${LINK} ufs_model.fd/FV3/upp upp.fd -fi +# TODO: Commenting out until UPP is up-to-date with Rocky-8. +#if [[ -d ufs_model.fd ]]; then +# [[ -d upp.fd ]] && rm -rf upp.fd +# ${LINK} ufs_model.fd/FV3/upp upp.fd +#fi if [[ -d gsi_enkf.fd ]]; then [[ -d gsi.fd ]] && rm -rf gsi.fd diff --git a/sorc/upp.fd b/sorc/upp.fd new file mode 160000 index 0000000000..41023fd972 --- /dev/null +++ b/sorc/upp.fd @@ -0,0 +1 @@ +Subproject commit 41023fd97241404050efa53f69c5f9bd31ac8b8c From a6a19241bc24409ab48678b158abacb074257f5d Mon Sep 17 00:00:00 2001 From: HenryWinterbottom-NOAA Date: Fri, 22 Mar 2024 14:35:07 +0000 Subject: [PATCH 51/63] Corrected UPP paths. --- sorc/build_upp.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/build_upp.sh b/sorc/build_upp.sh index e217e171db..1dca0035fd 100755 --- a/sorc/build_upp.sh +++ b/sorc/build_upp.sh @@ -26,6 +26,6 @@ if [[ ! -d "../exec" ]]; then mkdir -p ../exec fi -cd ufs_model.fd/FV3/upp/tests +cd upp.fd/tests # shellcheck disable=SC2086 BUILD_JOBS=${BUILD_JOBS:-8} ./compile_upp.sh ${_opts} From 4ee219284c4b9fec595dd7606ec4758ceb491808 Mon Sep 17 00:00:00 2001 From: HenryWinterbottom-NOAA Date: Fri, 22 Mar 2024 15:29:28 +0000 Subject: [PATCH 52/63] Updated hashes. --- sorc/gdas.cd | 2 +- sorc/upp.fd | 2 +- sorc/verif-global.fd | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index dd350d7e4d..e029bd7609 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit dd350d7e4daab0977407e388711807f13b204f6f +Subproject commit e029bd76095b5604f0d8cceb000f1d7c86e7530f diff --git a/sorc/upp.fd b/sorc/upp.fd index 41023fd972..15e2a8c1a5 160000 --- a/sorc/upp.fd +++ b/sorc/upp.fd @@ -1 +1 @@ -Subproject commit 41023fd97241404050efa53f69c5f9bd31ac8b8c +Subproject commit 15e2a8c1a5189621b51d948b3b8f88cf417bb620 diff --git a/sorc/verif-global.fd b/sorc/verif-global.fd index bd1c8f62a1..9377e84ba3 160000 --- a/sorc/verif-global.fd +++ b/sorc/verif-global.fd @@ -1 +1 @@ -Subproject commit bd1c8f62a1878051e34ff7c6f6a4dd290381f1ef +Subproject commit 9377e84ba3fc9b2fd13c2c84cfd571855dee75ae From 9ebdfd62b6dcea48aeedb384d6247c81ccb67a4f Mon Sep 17 00:00:00 2001 From: HenryWinterbottom-NOAA Date: Fri, 22 Mar 2024 17:26:06 +0000 Subject: [PATCH 53/63] CI/CD module load update. --- modulefiles/module_gwci.hera.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modulefiles/module_gwci.hera.lua b/modulefiles/module_gwci.hera.lua index 3d4c413a44..75b972b69b 100644 --- a/modulefiles/module_gwci.hera.lua +++ b/modulefiles/module_gwci.hera.lua @@ -2,7 +2,7 @@ help([[ Load environment to run GFS workflow setup scripts on Hera ]]) -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev/install/modulefiles/Core") +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev-rocky8/install/modulefiles/Core") load(pathJoin("stack-intel", os.getenv("2021.5.0"))) load(pathJoin("stack-intel-oneapi-mpi", os.getenv("2021.5.1"))) From c08dc7b85be0719cb67ba6fa690bfa92842864ec Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Sat, 23 Mar 2024 17:27:45 -0400 Subject: [PATCH 54/63] update gsi_enkf submodule pointer to develop --- sorc/gsi_enkf.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gsi_enkf.fd b/sorc/gsi_enkf.fd index f015cc4414..2167bc93fe 160000 --- a/sorc/gsi_enkf.fd +++ b/sorc/gsi_enkf.fd @@ -1 +1 @@ -Subproject commit f015cc441490684a4e47801b2b458893ef580477 +Subproject commit 2167bc93fe2b1e84657822a323666ccc61f9a339 From 10e84b6d3d86ff0b1b584ac5b6aa84fb7d4618ee Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Sat, 23 Mar 2024 17:29:50 -0400 Subject: [PATCH 55/63] update gitmodules to add upp and keep the rest same --- .gitmodules | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/.gitmodules b/.gitmodules index 934ef43d95..3a95d0ac50 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,30 +1,31 @@ +[submodule "sorc/ufs_model.fd"] + path = sorc/ufs_model.fd + url = https://github.com/ufs-community/ufs-weather-model + ignore = dirty [submodule "sorc/wxflow"] path = sorc/wxflow url = https://github.com/NOAA-EMC/wxflow +[submodule "sorc/gfs_utils.fd"] + path = sorc/gfs_utils.fd + url = https://github.com/NOAA-EMC/gfs-utils +[submodule "sorc/ufs_utils.fd"] + path = sorc/ufs_utils.fd + url = https://github.com/ufs-community/UFS_UTILS.git +[submodule "sorc/verif-global.fd"] + path = sorc/verif-global.fd + url = https://github.com/NOAA-EMC/EMC_verif-global.git [submodule "sorc/gsi_enkf.fd"] path = sorc/gsi_enkf.fd url = https://github.com/NOAA-EMC/GSI.git [submodule "sorc/gdas.cd"] path = sorc/gdas.cd url = https://github.com/NOAA-EMC/GDASApp.git -[submodule "sorc/verif-global.fd"] - path = sorc/verif-global.fd - url = https://github.com/NOAA-EMC/EMC_verif-global.git -[submodule "sorc/gsi_monitor.fd"] - path = sorc/gsi_monitor.fd - url = https://github.com/NOAA-EMC/GSI-Monitor -[submodule "sorc/gfs_utils.fd"] - path = sorc/gfs_utils.fd - url = https://github.com/NOAA-EMC/gfs-utils -[submodule "sorc/ufs_utils.fd"] - path = sorc/ufs_utils.fd - url = https://github.com/ufs-community/UFS_UTILS -[submodule "sorc/ufs_model.fd"] - path = sorc/ufs_model.fd - url = https://github.com/ufs-community/ufs-weather-model [submodule "sorc/gsi_utils.fd"] - path = sorc/gsi_utils.fd - url = https://github.com/NOAA-EMC/GSI-utils + path = sorc/gsi_utils.fd + url = https://github.com/NOAA-EMC/GSI-Utils.git +[submodule "sorc/gsi_monitor.fd"] + path = sorc/gsi_monitor.fd + url = https://github.com/NOAA-EMC/GSI-Monitor.git [submodule "sorc/upp.fd"] - path = sorc/upp.fd - url = https://github.com/NOAA-EMC/UPP + path = sorc/upp.fd + url = https://github.com/NOAA-EMC/UPP From b5bf19712b5a2e815337ce7cbe9d7a4a853074c3 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Sat, 23 Mar 2024 17:43:56 -0400 Subject: [PATCH 56/63] make reviewer updates --- modulefiles/module_base.hera.lua | 10 +++++++--- sorc/link_workflow.sh | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/modulefiles/module_base.hera.lua b/modulefiles/module_base.hera.lua index 7510d7f6cd..ed1ef1b5fc 100644 --- a/modulefiles/module_base.hera.lua +++ b/modulefiles/module_base.hera.lua @@ -41,10 +41,14 @@ load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None"))) setenv("WGRIB2","wgrib2") setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) -prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/feature-hera-rocky8/modulefiles")) +--prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) +prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/dev-gfsv17/modulefiles")) load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None"))) -prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/Fit2Obs/feature-hera-rocky8/modulefiles")) -load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None"))) +--prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) +--load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None"))) +-- TODO: After v1.1.0 is installed on Orion/WCOSS2/Hercules and run.spack.ver is updated, the following 2 lines can be removed and the above uncommented +prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/Fit2Obs/v1.1.0/modulefiles")) +load(pathJoin("fit2obs", "1.1.0")) whatis("Description: GFS run environment") diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index ee749f4d51..b052b0b1c1 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -345,7 +345,7 @@ fi #--link source code directories #------------------------------ cd "${HOMEgfs}/sorc" || exit 8 -# TODO: Commenting out until UPP is up-to-date with Rocky-8. +# TODO: Commenting out until UPP is up-to-date with Rocky-8 in the UFSWM. #if [[ -d ufs_model.fd ]]; then # [[ -d upp.fd ]] && rm -rf upp.fd # ${LINK} ufs_model.fd/FV3/upp upp.fd From fe57ef590640abd81caf8865249f8c8bec556bac Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Sat, 23 Mar 2024 19:06:34 -0400 Subject: [PATCH 57/63] update upp submodule to develop --- sorc/upp.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/upp.fd b/sorc/upp.fd index 15e2a8c1a5..e2189f7d0d 160000 --- a/sorc/upp.fd +++ b/sorc/upp.fd @@ -1 +1 @@ -Subproject commit 15e2a8c1a5189621b51d948b3b8f88cf417bb620 +Subproject commit e2189f7d0dcaa3ce15050618a3eeb2c4ef8774be From 243f20bc3af3b3f21baac974af29e8f542d70a83 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Sat, 23 Mar 2024 20:12:36 -0400 Subject: [PATCH 58/63] update upp version to develop and set fit2obs_ver in module_base.hera --- modulefiles/module_base.hera.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modulefiles/module_base.hera.lua b/modulefiles/module_base.hera.lua index ed1ef1b5fc..e2cf721eb4 100644 --- a/modulefiles/module_base.hera.lua +++ b/modulefiles/module_base.hera.lua @@ -49,6 +49,6 @@ load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None"))) --load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None"))) -- TODO: After v1.1.0 is installed on Orion/WCOSS2/Hercules and run.spack.ver is updated, the following 2 lines can be removed and the above uncommented prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/Fit2Obs/v1.1.0/modulefiles")) -load(pathJoin("fit2obs", "1.1.0")) +load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None"))) whatis("Description: GFS run environment") From fe5b0cec53b65abb960a76a97d29b426cf2cd1e5 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Sat, 23 Mar 2024 20:27:36 -0400 Subject: [PATCH 59/63] straighten out the comment for fit2obs modulefile on hera --- modulefiles/module_base.hera.lua | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/modulefiles/module_base.hera.lua b/modulefiles/module_base.hera.lua index e2cf721eb4..aece027afa 100644 --- a/modulefiles/module_base.hera.lua +++ b/modulefiles/module_base.hera.lua @@ -46,8 +46,10 @@ prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepob load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None"))) --prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) ---load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None"))) --- TODO: After v1.1.0 is installed on Orion/WCOSS2/Hercules and run.spack.ver is updated, the following 2 lines can be removed and the above uncommented +-- TODO: After v1.1.0 is installed on Orion/WCOSS2/Hercules and run.spack.ver is updated, the following lines can be removed and the above uncommented +-- Also, the modulefile for v1.1.0 needs to be created for Hera under +-- /scratch1/NCEPDEV/global/glopara/git/Fit2Obs/v1.1.0/modulefiles/fit2obs +-- currently there is only one for v1.0.0 prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/Fit2Obs/v1.1.0/modulefiles")) load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None"))) From faadb532717533abae34f383dedeb650c10e8ca7 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Sat, 23 Mar 2024 21:39:02 -0400 Subject: [PATCH 60/63] pip is not found in load_ufsda_modules.sh. Remove pip list from there --- ush/load_ufsda_modules.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/load_ufsda_modules.sh b/ush/load_ufsda_modules.sh index e8e72b8fbe..6bab04b3bf 100755 --- a/ush/load_ufsda_modules.sh +++ b/ush/load_ufsda_modules.sh @@ -42,14 +42,14 @@ case "${MACHINE_ID}" in ;; ("wcoss2" | "acorn" | "jet" | "gaea" | "s4") echo WARNING: UFSDA NOT SUPPORTED ON THIS PLATFORM - ;; + ;; *) echo "WARNING: UNKNOWN PLATFORM" ;; esac module list -pip list +#pip list # Restore stack soft limit: ulimit -S -s "${ulimit_s}" From 9556d4a002e5645cc9aa7a37d274609588c1c5cb Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Sat, 23 Mar 2024 21:50:33 -0400 Subject: [PATCH 61/63] update path to iodaconv and python version --- jobs/rocoto/prepsnowobs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/rocoto/prepsnowobs.sh b/jobs/rocoto/prepsnowobs.sh index dae11129eb..d260eef46c 100755 --- a/jobs/rocoto/prepsnowobs.sh +++ b/jobs/rocoto/prepsnowobs.sh @@ -14,7 +14,7 @@ export jobid="${job}.$$" ############################################################### # setup python path for workflow utilities and tasks wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src" -gdasappPATH="${HOMEgfs}/sorc/gdas.cd/iodaconv/src:${HOMEgfs}/sorc/gdas.cd/build/lib/python3.7" +gdasappPATH="${HOMEgfs}/sorc/gdas.cd/bundle/iodaconv/src:${HOMEgfs}/sorc/gdas.cd/build/lib/python3.10" PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}:${gdasappPATH}" export PYTHONPATH From 6a9f03dec117d8d354c00b356e8b9583dd4663b6 Mon Sep 17 00:00:00 2001 From: Kate Friedman Date: Mon, 25 Mar 2024 10:09:06 -0400 Subject: [PATCH 62/63] Update fit2obs_ver in run.spack.ver The new Fit2Obs tag is v1.1.0. Change fit2obs_ver=1.1.0 Refs #2329 --- versions/run.spack.ver | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versions/run.spack.ver b/versions/run.spack.ver index c00b7483cd..412e8c93ea 100644 --- a/versions/run.spack.ver +++ b/versions/run.spack.ver @@ -31,4 +31,4 @@ export obsproc_run_ver=1.1.2 export prepobs_run_ver=1.0.1 export ens_tracker_ver=feature-GFSv17_com_reorg -export fit2obs_ver=1.0.0 +export fit2obs_ver=1.1.0 From 79168db467b8d3324e6c22d3815a3bf5b692667c Mon Sep 17 00:00:00 2001 From: Kate Friedman Date: Mon, 25 Mar 2024 10:10:59 -0400 Subject: [PATCH 63/63] Update Fit2Obs path in module_base.hera.lua Set the Fit2Obs MODULEPATH back to use fit2obs_ver. Refs #2329 --- modulefiles/module_base.hera.lua | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/modulefiles/module_base.hera.lua b/modulefiles/module_base.hera.lua index aece027afa..1a94019d65 100644 --- a/modulefiles/module_base.hera.lua +++ b/modulefiles/module_base.hera.lua @@ -45,12 +45,7 @@ setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/dev-gfsv17/modulefiles")) load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None"))) ---prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) --- TODO: After v1.1.0 is installed on Orion/WCOSS2/Hercules and run.spack.ver is updated, the following lines can be removed and the above uncommented --- Also, the modulefile for v1.1.0 needs to be created for Hera under --- /scratch1/NCEPDEV/global/glopara/git/Fit2Obs/v1.1.0/modulefiles/fit2obs --- currently there is only one for v1.0.0 -prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/Fit2Obs/v1.1.0/modulefiles")) +prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None"))) whatis("Description: GFS run environment")