diff --git a/.gitignore b/.gitignore index 3eb6853c19..3abc6e6449 100644 --- a/.gitignore +++ b/.gitignore @@ -142,6 +142,7 @@ ush/make_ntc_bull.pl ush/make_tif.sh ush/month_name.sh ush/imsfv3_scf2ioda.py +ush/atparse.bash # version files versions/build.ver diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources index e5995b3cf2..6f1db0007b 100644 --- a/parm/config/gefs/config.resources +++ b/parm/config/gefs/config.resources @@ -550,38 +550,42 @@ elif [[ "${step}" = "fcst" || "${step}" = "efcs" ]]; then export MEDPETS MEDTHREADS echo "MEDIATOR using (threads, PETS) = (${MEDTHREADS}, ${MEDPETS})" + CHMPETS=0; CHMTHREADS=0 if [[ "${DO_AERO}" = "YES" ]]; then # GOCART shares the same grid and forecast tasks as FV3 (do not add write grid component tasks). (( CHMTHREADS = ATMTHREADS )) (( CHMPETS = FV3PETS )) # Do not add to NTASKS_TOT - export CHMPETS CHMTHREADS echo "GOCART using (threads, PETS) = (${CHMTHREADS}, ${CHMPETS})" fi + export CHMPETS CHMTHREADS + WAVPETS=0; WAVTHREADS=0 if [[ "${DO_WAVE}" = "YES" ]]; then (( WAVPETS = ntasks_ww3 * nthreads_ww3 )) (( WAVTHREADS = nthreads_ww3 )) - export WAVPETS WAVTHREADS echo "WW3 using (threads, PETS) = (${WAVTHREADS}, ${WAVPETS})" (( NTASKS_TOT = NTASKS_TOT + WAVPETS )) fi + export WAVPETS WAVTHREADS + OCNPETS=0; OCNTHREADS=0 if [[ "${DO_OCN}" = "YES" ]]; then (( OCNPETS = ntasks_mom6 * nthreads_mom6 )) (( OCNTHREADS = nthreads_mom6 )) - export OCNPETS OCNTHREADS echo "MOM6 using (threads, PETS) = (${OCNTHREADS}, ${OCNPETS})" (( NTASKS_TOT = NTASKS_TOT + OCNPETS )) fi + export OCNPETS OCNTHREADS + ICEPETS=0; ICETHREADS=0 if [[ "${DO_ICE}" = "YES" ]]; then (( ICEPETS = ntasks_cice6 * nthreads_cice6 )) (( ICETHREADS = nthreads_cice6 )) - export ICEPETS ICETHREADS echo "CICE6 using (threads, PETS) = (${ICETHREADS}, ${ICEPETS})" (( NTASKS_TOT = NTASKS_TOT + ICEPETS )) fi + export ICEPETS ICETHREADS echo "Total PETS for ${_CDUMP} = ${NTASKS_TOT}" diff --git a/parm/config/gefs/config.ufs b/parm/config/gefs/config.ufs index c23ccf2abd..bfbb0a12cb 100644 --- a/parm/config/gefs/config.ufs +++ b/parm/config/gefs/config.ufs @@ -255,6 +255,7 @@ if [[ "${skip_mom6}" == "false" ]]; then CHLCLIM="seawifs_1998-2006_smoothed_2X.nc" MOM6_RESTART_SETTING='r' MOM6_RIVER_RUNOFF='False' + eps_imesh="4.0e-1" ;; "100") ntasks_mom6=20 @@ -267,6 +268,7 @@ if [[ "${skip_mom6}" == "false" ]]; then CHLCLIM="seawifs_1998-2006_smoothed_2X.nc" MOM6_RESTART_SETTING='n' MOM6_RIVER_RUNOFF='False' + eps_imesh="2.5e-1" ;; "50") ntasks_mom6=60 @@ -279,7 +281,8 @@ if [[ "${skip_mom6}" == "false" ]]; then CHLCLIM="seawifs-clim-1997-2010.${NX_GLB}x${NY_GLB}.v20180328.nc" MOM6_RESTART_SETTING='n' MOM6_RIVER_RUNOFF='True' - ;; + eps_imesh="1.0e-1" + ;; "025") ntasks_mom6=220 OCNTIM=1800 @@ -291,12 +294,14 @@ if [[ "${skip_mom6}" == "false" ]]; then CHLCLIM="seawifs-clim-1997-2010.${NX_GLB}x${NY_GLB}.v20180328.nc" MOM6_RIVER_RUNOFF='True' MOM6_RESTART_SETTING="r" + eps_imesh="1.0e-1" ;; *) echo "FATAL ERROR: Unsupported MOM6 resolution = ${mom6_res}, ABORT!" exit 1 ;; esac + export nthreads_mom6 ntasks_mom6 export OCNTIM export NX_GLB NY_GLB @@ -305,6 +310,7 @@ if [[ "${skip_mom6}" == "false" ]]; then export CHLCLIM export MOM6_RIVER_RUNOFF export MOM6_RESTART_SETTING + export eps_imesh fi # CICE6 specific settings @@ -357,10 +363,12 @@ if [[ "${skip_ww3}" == "false" ]]; then ntasks_ww3=262 ;; "glo_200") - ntasks_ww3=40 + ntasks_ww3=30 + nthreads_ww3=1 ;; "glo_500") - ntasks_ww3=10 + ntasks_ww3=12 + nthreads_ww3=1 ;; "mx025") ntasks_ww3=80 diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 2cc8b2426f..bc9edfcb7e 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -544,38 +544,42 @@ elif [[ "${step}" = "fcst" || "${step}" = "efcs" ]]; then export MEDPETS MEDTHREADS echo "MEDIATOR using (threads, PETS) = (${MEDTHREADS}, ${MEDPETS})" + CHMPETS=0; CHMTHREADS=0 if [[ "${DO_AERO}" = "YES" ]]; then # GOCART shares the same grid and forecast tasks as FV3 (do not add write grid component tasks). (( CHMTHREADS = ATMTHREADS )) (( CHMPETS = FV3PETS )) # Do not add to NTASKS_TOT - export CHMPETS CHMTHREADS echo "GOCART using (threads, PETS) = (${CHMTHREADS}, ${CHMPETS})" fi + export CHMPETS CHMTHREADS + WAVPETS=0; WAVTHREADS=0 if [[ "${DO_WAVE}" = "YES" ]]; then (( WAVPETS = ntasks_ww3 * nthreads_ww3 )) (( WAVTHREADS = nthreads_ww3 )) - export WAVPETS WAVTHREADS echo "WW3 using (threads, PETS) = (${WAVTHREADS}, ${WAVPETS})" (( NTASKS_TOT = NTASKS_TOT + WAVPETS )) fi + export WAVPETS WAVTHREADS + OCNPETS=0; OCNTHREADS=0 if [[ "${DO_OCN}" = "YES" ]]; then (( OCNPETS = ntasks_mom6 * nthreads_mom6 )) (( OCNTHREADS = nthreads_mom6 )) - export OCNPETS OCNTHREADS echo "MOM6 using (threads, PETS) = (${OCNTHREADS}, ${OCNPETS})" (( NTASKS_TOT = NTASKS_TOT + OCNPETS )) fi + export OCNPETS OCNTHREADS + ICEPETS=0; ICETHREADS=0 if [[ "${DO_ICE}" = "YES" ]]; then (( ICEPETS = ntasks_cice6 * nthreads_cice6 )) (( ICETHREADS = nthreads_cice6 )) - export ICEPETS ICETHREADS echo "CICE6 using (threads, PETS) = (${ICETHREADS}, ${ICEPETS})" (( NTASKS_TOT = NTASKS_TOT + ICEPETS )) fi + export ICEPETS ICETHREADS echo "Total PETS for ${_CDUMP} = ${NTASKS_TOT}" diff --git a/parm/config/gfs/config.ufs b/parm/config/gfs/config.ufs index 1ec419b822..bfbb0a12cb 100644 --- a/parm/config/gfs/config.ufs +++ b/parm/config/gfs/config.ufs @@ -255,6 +255,7 @@ if [[ "${skip_mom6}" == "false" ]]; then CHLCLIM="seawifs_1998-2006_smoothed_2X.nc" MOM6_RESTART_SETTING='r' MOM6_RIVER_RUNOFF='False' + eps_imesh="4.0e-1" ;; "100") ntasks_mom6=20 @@ -267,6 +268,7 @@ if [[ "${skip_mom6}" == "false" ]]; then CHLCLIM="seawifs_1998-2006_smoothed_2X.nc" MOM6_RESTART_SETTING='n' MOM6_RIVER_RUNOFF='False' + eps_imesh="2.5e-1" ;; "50") ntasks_mom6=60 @@ -279,7 +281,8 @@ if [[ "${skip_mom6}" == "false" ]]; then CHLCLIM="seawifs-clim-1997-2010.${NX_GLB}x${NY_GLB}.v20180328.nc" MOM6_RESTART_SETTING='n' MOM6_RIVER_RUNOFF='True' - ;; + eps_imesh="1.0e-1" + ;; "025") ntasks_mom6=220 OCNTIM=1800 @@ -291,12 +294,14 @@ if [[ "${skip_mom6}" == "false" ]]; then CHLCLIM="seawifs-clim-1997-2010.${NX_GLB}x${NY_GLB}.v20180328.nc" MOM6_RIVER_RUNOFF='True' MOM6_RESTART_SETTING="r" + eps_imesh="1.0e-1" ;; *) echo "FATAL ERROR: Unsupported MOM6 resolution = ${mom6_res}, ABORT!" exit 1 ;; esac + export nthreads_mom6 ntasks_mom6 export OCNTIM export NX_GLB NY_GLB @@ -305,6 +310,7 @@ if [[ "${skip_mom6}" == "false" ]]; then export CHLCLIM export MOM6_RIVER_RUNOFF export MOM6_RESTART_SETTING + export eps_imesh fi # CICE6 specific settings diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index f503e7e273..36afb4d4c1 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -154,6 +154,30 @@ for file in finddate.sh make_ntc_bull.pl make_NTC_file.pl make_tif.sh month_name ${LINK_OR_COPY} "${HOMEgfs}/sorc/gfs_utils.fd/ush/${file}" . done +# TODO: Link these nems.configure templates from ufs-weather-model +#cd "${HOMEgfs}/parm/ufs" || exit 1 +#declare -a nems_configure_files=("nems.configure.atm.IN" \ +# "nems.configure.atm_aero.IN" \ +# "nems.configure.atmw.IN" \ +# "nems.configure.blocked_atm_wav_2way.IN" \ +# "nems.configure.blocked_atm_wav.IN" \ +# "nems.configure.cpld_agrid.IN" \ +# "nems.configure.cpld_esmfthreads.IN" \ +# "nems.configure.cpld.IN" \ +# "nems.configure.cpld_noaero.IN" \ +# "nems.configure.cpld_noaero_nowave.IN" \ +# "nems.configure.cpld_noaero_outwav.IN" \ +# "nems.configure.leapfrog_atm_wav.IN") +#for file in "${nems_configure_files[@]}"; do +# [[ -s "${file}" ]] && rm -f "${file}" +# ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/tests/parm/${file}" . +#done + +cd "${HOMEgfs}/ush" || exit 1 +[[ -s "atparse.bash" ]] && rm -f "atparse.bash" +${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/tests/atparse.bash" . + + #------------------------------ #--add GDASApp fix directory #------------------------------ diff --git a/ush/nems_configure.sh b/ush/nems_configure.sh index c51e500d5d..1c62cdfd25 100755 --- a/ush/nems_configure.sh +++ b/ush/nems_configure.sh @@ -39,153 +39,68 @@ else exit 1 fi +local atm_model="fv3" local atm_petlist_bounds="0 $(( ${ATMPETS}-1 ))" -local med_petlist_bounds="0 $(( ${MEDPETS}-1 ))" +local atm_omp_num_threads="${ATMTHREADS}" -sed -i -e "s;@\[atm_model\];fv3;g" tmp1 -sed -i -e "s;@\[atm_petlist_bounds\];${atm_petlist_bounds};g" tmp1 -sed -i -e "s;@\[atm_omp_num_threads\];${ATMTHREADS};g" tmp1 -sed -i -e "s;@\[med_model\];cmeps;g" tmp1 -sed -i -e "s;@\[med_petlist_bounds\];${med_petlist_bounds};g" tmp1 -sed -i -e "s;@\[med_omp_num_threads\];${MEDTHREADS};g" tmp1 -sed -i -e "s;@\[esmf_logkind\];${esmf_logkind};g" tmp1 +local med_model="cmeps" +local med_petlist_bounds="0 $(( ${MEDPETS}-1 ))" +local med_omp_num_threads="${MEDTHREADS}" if [[ "${cpl}" = ".true." ]]; then - sed -i -e "s;@\[coupling_interval_slow_sec\];${CPL_SLOW};g" tmp1 + local coupling_interval_slow_sec="${CPL_SLOW}" fi if [[ "${cplflx}" = ".true." ]]; then - # TODO: Should this be raised up to config.ufs or config.ocn? - case "${OCNRES}" in - "500") local eps_imesh="4.0e-1";; - "100") local eps_imesh="2.5e-1";; - *) local eps_imesh="1.0e-1";; - esac local use_coldstart=${use_coldstart:-".false."} local use_mommesh=${USE_MOMMESH:-"true"} - local restile=$(echo "${CASE}" |cut -c2-) - - local start="${ATMPETS}" - local end="$(( ${start}+${OCNPETS}-1 ))" - local ocn_petlist_bounds="${start} ${end}" - - sed -i -e "s;@\[ocn_model\];mom6;g" tmp1 - sed -i -e "s;@\[ocn_petlist_bounds\];${ocn_petlist_bounds};g" tmp1 - sed -i -e "s;@\[ocn_omp_num_threads\];${OCNTHREADS};g" tmp1 - sed -i -e "s;@\[DumpFields\];${DumpFields};g" tmp1 - sed -i -e "s;@\[cap_dbug_flag\];${cap_dbug_flag};g" tmp1 - sed -i -e "s;@\[use_coldstart\];${use_coldstart};g" tmp1 - sed -i -e "s;@\[RUNTYPE\];${cmeps_run_type};g" tmp1 - sed -i -e "s;@\[CPLMODE\];${cplmode};g" tmp1 - sed -i -e "s;@\[coupling_interval_fast_sec\];${CPL_FAST};g" tmp1 - sed -i -e "s;@\[RESTART_N\];${restart_interval};g" tmp1 - sed -i -e "s;@\[use_mommesh\];${use_mommesh};g" tmp1 - sed -i -e "s;@\[eps_imesh\];${eps_imesh};g" tmp1 - sed -i -e "s;@\[ATMTILESIZE\];${restile};g" tmp1 + + local ocn_model="mom6" + local ocn_petlist_bounds="${ATMPETS} $(( ${ATMPETS}+${OCNPETS}-1 ))" + local ocn_omp_num_threads="${OCNTHREADS}" + local RUNTYPE="${cmeps_run_type}" + local CPLMODE="${cplmode}" + local coupling_interval_fast_sec="${CPL_FAST}" + local RESTART_N="${restart_interval}" + local ATMTILESIZE="${CASE:2:}" fi if [[ "${cplice}" = ".true." ]]; then - local mesh_ocn_ice=${MESH_OCN_ICE:-"mesh.mx${ICERES}.nc"} - - local start="$(( ${ATMPETS}+${OCNPETS} ))" - local end="$(( ${start}+${ICEPETS}-1 ))" - local ice_petlist_bounds="${start} ${end}" - - sed -i -e "s;@\[ice_model\];cice6;g" tmp1 - sed -i -e "s;@\[ice_petlist_bounds\];${ice_petlist_bounds};g" tmp1 - sed -i -e "s;@\[ice_omp_num_threads\];${ICETHREADS};g" tmp1 - sed -i -e "s;@\[MESH_OCN_ICE\];${mesh_ocn_ice};g" tmp1 - sed -i -e "s;@\[FHMAX\];${FHMAX_GFS};g" tmp1 + local ice_model="cice6" + local ice_petlist_bounds="$(( ${ATMPETS}+${OCNPETS} )) $(( ${ATMPETS}+${OCNPETS}+${ICEPETS}-1 ))" + local ice_omp_num_threads="${ICETHREADS}" + local MESH_OCN_ICE=${MESH_OCN_ICE:-"mesh.mx${ICERES}.nc"} + local FHMAX="${FHMAX_GFS}" # TODO: How did this get in here hard-wired to FHMAX_GFS? fi if [[ "${cplwav}" = ".true." ]]; then - local start="$(( ${ATMPETS}+${OCNPETS:-0}+${ICEPETS:-0} ))" - local end="$(( ${start}+${WAVPETS}-1 ))" - local wav_petlist_bounds="${start} ${end}" + local wav_model="ww3" + local wav_petlist_bounds="$(( ${ATMPETS}+${OCNPETS}+${ICEPETS} )) $(( ${ATMPETS}+${OCNPETS}+${ICEPETS}+${WAVPETS}-1 ))" + local wav_omp_num_threads="${WAVTHREADS}" + local MULTIGRID="${waveMULTIGRID}" - sed -i -e "s;@\[wav_model\];ww3;g" tmp1 - sed -i -e "s;@\[wav_petlist_bounds\];${wav_petlist_bounds};g" tmp1 - sed -i -e "s;@\[wav_omp_num_threads\];${WAVTHREADS};g" tmp1 - sed -i -e "s;@\[MESH_WAV\];${MESH_WAV};g" tmp1 - sed -i -e "s;@\[MULTIGRID\];${waveMULTIGRID};g" tmp1 fi if [[ "${cplchm}" = ".true." ]]; then + local chm_model="gocart" local chm_petlist_bounds="0 $(( ${CHMPETS}-1 ))" + local chm_omp_num_threads="${CHMTHREADS}" + local coupling_interval_fast_sec="${CPL_FAST}" - sed -i -e "s;@\[chm_model\];gocart;g" tmp1 - sed -i -e "s;@\[chm_petlist_bounds\];${chm_petlist_bounds};g" tmp1 - sed -i -e "s;@\[chm_omp_num_threads\];${CHMTHREADS};g" tmp1 - sed -i -e "s;@\[coupling_interval_fast_sec\];${CPL_FAST};g" tmp1 fi -mv tmp1 nems.configure +source "${HOMEgfs}/ush/atparse.bash" +rm -f "${DATA}/nems.configure" +atparse < "${infile}" >> "${DATA}/nems.configure" echo "$(cat nems.configure)" -if [[ "${cplflx}" = ".true." ]]; then - -#Create other CMEPS mediator related files -cat > pio_in << EOF -&papi_inparm - papi_ctr1_str = "PAPI_FP_OPS" - papi_ctr2_str = "PAPI_NO_CTR" - papi_ctr3_str = "PAPI_NO_CTR" - papi_ctr4_str = "PAPI_NO_CTR" -/ -&pio_default_inparm - pio_async_interface = .false. - pio_blocksize = -1 - pio_buffer_size_limit = -1 - pio_debug_level = 0 - pio_rearr_comm_enable_hs_comp2io = .true. - pio_rearr_comm_enable_hs_io2comp = .false. - pio_rearr_comm_enable_isend_comp2io = .false. - pio_rearr_comm_enable_isend_io2comp = .true. - pio_rearr_comm_fcd = "2denable" - pio_rearr_comm_max_pend_req_comp2io = 0 - pio_rearr_comm_max_pend_req_io2comp = 64 - pio_rearr_comm_type = "p2p" -/ -&prof_inparm - profile_add_detail = .false. - profile_barrier = .false. - profile_depth_limit = 4 - profile_detail_limit = 2 - profile_disable = .false. - profile_global_stats = .true. - profile_outpe_num = 1 - profile_outpe_stride = 0 - profile_ovhd_measurement = .false. - profile_papi_enable = .false. - profile_single_file = .false. - profile_timer = 4 -/ -EOF - -echo "$(cat pio_in)" - -cat > med_modelio.nml << EOF -&pio_inparm - pio_netcdf_format = "64bit_offset" - pio_numiotasks = -99 - pio_rearranger = 1 - pio_root = 1 - pio_stride = 36 - pio_typename = "netcdf" -/ -EOF - -echo "$(cat med_modelio.nml)" - -fi - ${NCP} "${HOMEgfs}/sorc/ufs_model.fd/tests/parm/fd_nems.yaml" fd_nems.yaml -echo "SUB ${FUNCNAME[0]}: Nems configured for ${confignamevarfornems}" +echo "SUB ${FUNCNAME[0]}: nems.configure.sh ends for ${confignamevarfornems}" }