From cf83885548bb3a6740c033f42479ce2ad283a4a9 Mon Sep 17 00:00:00 2001 From: Jessica Meixner Date: Fri, 16 Feb 2024 01:55:02 -0500 Subject: [PATCH] Add unstructured grid for HR3/GFS (#2230) This adds the capability to use unstructured grids in the global workflow, which will be used in HR3. There are new fix files for a low-resolution 100km grid and a grid closer to our targeted GFSv17 grid which has the resolution combined from the older multi_1 and GFSv16 grids. The fix file update is here: NOAA-EMC/global-workflow#2229 Note: This now means that GFS tests need a new build option: `./build_all.sh -w` So that PDLIB=ON is turned on for compiling relevant UFS and WW3 codes. Resolves NOAA-EMC/global-workflow#1547 --- ci/cases/pr/C48_S2SWA_gefs.yaml | 5 ++++ ci/cases/pr/C48mx500_3DVarAOWCDA.yaml | 2 +- ci/cases/yamls/build.yaml | 2 +- ci/scripts/clone-build_ci.sh | 2 +- docs/source/clone.rst | 7 ++++++ env/WCOSS2.env | 2 ++ parm/config/gfs/config.base.emc.dyn | 8 +++---- parm/config/gfs/config.resources | 4 ++-- parm/config/gfs/config.stage_ic | 16 ++++++------- parm/config/gfs/config.ufs | 10 +++++++- parm/config/gfs/config.wave | 14 +++++++++++- parm/wave/at_10m_interp.inp.tmpl | 2 +- parm/wave/ep_10m_interp.inp.tmpl | 2 +- parm/wave/glo_15mxt_interp.inp.tmpl | 6 ++--- parm/wave/glo_200_interp.inp.tmpl | 12 ++++++++++ parm/wave/glo_30m_interp.inp.tmpl | 6 ++--- parm/wave/wc_10m_interp.inp.tmpl | 2 +- scripts/exgfs_wave_init.sh | 13 ++++++++--- sorc/build_all.sh | 14 ++++++++---- sorc/build_ufs.sh | 4 +++- sorc/build_ww3prepost.sh | 33 +++++++++++++++++++-------- ush/wave_grid_moddef.sh | 16 +++++++++---- versions/fix.ver | 2 +- 23 files changed, 130 insertions(+), 54 deletions(-) create mode 100755 parm/wave/glo_200_interp.inp.tmpl diff --git a/ci/cases/pr/C48_S2SWA_gefs.yaml b/ci/cases/pr/C48_S2SWA_gefs.yaml index d42f4cd15b..de677a70b6 100644 --- a/ci/cases/pr/C48_S2SWA_gefs.yaml +++ b/ci/cases/pr/C48_S2SWA_gefs.yaml @@ -16,3 +16,8 @@ arguments: idate: 2021032312 edate: 2021032312 yaml: {{ HOMEgfs }}/ci/cases/yamls/gefs_ci_defaults.yaml + +skip_ci_on_hosts: + - hera + - orion + - hercules diff --git a/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml b/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml index 6e9fc6d3de..d9156e38f3 100644 --- a/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml +++ b/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml @@ -17,7 +17,7 @@ arguments: start: warm yaml: {{ HOMEgfs }}/ci/cases/yamls/soca_gfs_defaults_ci.yaml -skip_ci_on_host: +skip_ci_on_hosts: - orion - hera - hercules diff --git a/ci/cases/yamls/build.yaml b/ci/cases/yamls/build.yaml index 5398fa1889..2ff008d372 100644 --- a/ci/cases/yamls/build.yaml +++ b/ci/cases/yamls/build.yaml @@ -1,3 +1,3 @@ builds: - gefs: './build_all.sh' - - gfs: './build_all.sh -gu' \ No newline at end of file + - gfs: './build_all.sh -wgu' diff --git a/ci/scripts/clone-build_ci.sh b/ci/scripts/clone-build_ci.sh index 798c98bf50..989afabb80 100755 --- a/ci/scripts/clone-build_ci.sh +++ b/ci/scripts/clone-build_ci.sh @@ -74,7 +74,7 @@ set +e source "${HOMEgfs}/ush/module-setup.sh" export BUILD_JOBS=8 rm -rf log.build -./build_all.sh -gu >> log.build 2>&1 +./build_all.sh -guw >> log.build 2>&1 build_status=$? DATE=$(date +'%D %r') diff --git a/docs/source/clone.rst b/docs/source/clone.rst index bad3f0e9f6..4f47eb230f 100644 --- a/docs/source/clone.rst +++ b/docs/source/clone.rst @@ -39,6 +39,13 @@ For coupled cycling (include new UFSDA) use the `-gu` options during build: ./build_all.sh -gu +For building with PDLIB for the wave model, use the `-w` options during build: + +:: + + ./build_all.sh -w + + Build workflow components and link workflow artifacts such as executables, etc. :: diff --git a/env/WCOSS2.env b/env/WCOSS2.env index bbf4de2ae3..e247a37bc9 100755 --- a/env/WCOSS2.env +++ b/env/WCOSS2.env @@ -176,6 +176,8 @@ elif [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then if [[ "${step}" = "fcst" ]]; then export OMP_PLACES=cores export OMP_STACKSIZE=2048M + export MPICH_MPIIO_HINTS="*:romio_cb_write=disable" + export FI_OFI_RXM_SAR_LIMIT=3145728 elif [[ "${step}" = "efcs" ]]; then export MPICH_MPIIO_HINTS="*:romio_cb_write=disable" export FI_OFI_RXM_SAR_LIMIT=3145728 diff --git a/parm/config/gfs/config.base.emc.dyn b/parm/config/gfs/config.base.emc.dyn index 48b66ab1b1..4301679fbb 100644 --- a/parm/config/gfs/config.base.emc.dyn +++ b/parm/config/gfs/config.base.emc.dyn @@ -183,16 +183,16 @@ export ICERES="${OCNRES}" # These are the currently recommended grid-combinations case "${CASE}" in "C48") - export waveGRD='glo_500' + export waveGRD='uglo_100km' ;; "C96" | "C192") - export waveGRD='glo_200' + export waveGRD='uglo_100km' ;; "C384") - export waveGRD='glo_025' + export waveGRD='uglo_100km' ;; "C768" | "C1152") - export waveGRD='mx025' + export waveGRD='uglo_m1g16' ;; *) echo "FATAL ERROR: Unrecognized CASE ${CASE}, ABORT!" diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index ced6e6a3d8..afebc70ae2 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -737,7 +737,7 @@ case ${step} in export npe_genesis=1 export nth_genesis=1 export npe_node_genesis=1 - export memory_genesis="4G" + export memory_genesis="10G" ;; "genesis_fsu") @@ -745,7 +745,7 @@ case ${step} in export npe_genesis_fsu=1 export nth_genesis_fsu=1 export npe_node_genesis_fsu=1 - export memory_genesis_fsu="4G" + export memory_genesis_fsu="10G" ;; "fit2obs") diff --git a/parm/config/gfs/config.stage_ic b/parm/config/gfs/config.stage_ic index 7f3956af4d..dc224b72db 100644 --- a/parm/config/gfs/config.stage_ic +++ b/parm/config/gfs/config.stage_ic @@ -21,16 +21,16 @@ case "${CASE}" in export CPL_WAVIC=workflow_C384_refactored ;; "C768") - export CPL_ATMIC=HR2_refactored - export CPL_ICEIC=HR1_refactored - export CPL_OCNIC=HR1_refactored - export CPL_WAVIC=HR1_refactored + export CPL_ATMIC=HR3C768 + export CPL_ICEIC=HR3marine + export CPL_OCNIC=HR3marine + export CPL_WAVIC=HR3marine ;; "C1152") - export CPL_ATMIC=HR2_C1152_refactored - export CPL_ICEIC=HR3_refactored - export CPL_OCNIC=HR3_refactored - export CPL_WAVIC=HR1_refactored + export CPL_ATMIC=HR3C1152 + export CPL_ICEIC=HR3marine + export CPL_OCNIC=HR3marine + export CPL_WAVIC=HR3marine ;; *) echo "FATAL ERROR Unrecognized resolution: ${CASE}" diff --git a/parm/config/gfs/config.ufs b/parm/config/gfs/config.ufs index c8ce216899..2e299fddf7 100644 --- a/parm/config/gfs/config.ufs +++ b/parm/config/gfs/config.ufs @@ -15,7 +15,7 @@ if (( $# <= 1 )); then echo "--fv3 C48|C96|C192|C384|C768|C1152|C3072" echo "--mom6 500|100|025" echo "--cice6 500|100|025" - echo "--ww3 gnh_10m;aoc_9km;gsh_15m|gwes_30m|glo_025|glo_200|glo_500|mx025" + echo "--ww3 gnh_10m;aoc_9km;gsh_15m|gwes_30m|glo_025|glo_200|glo_500|mx025|uglo_100km|uglo_m1g16" echo "--gocart" exit 1 @@ -416,6 +416,14 @@ if [[ "${skip_ww3}" == "false" ]]; then "mx025") ntasks_ww3=80 ;; + "uglo_100km") + ntasks_ww3=40 + nthreads_ww3=1 + ;; + "uglo_m1g16") + ntasks_ww3=1000 + nthreads_ww3=1 + ;; *) echo "FATAL ERROR: Unsupported WW3 resolution = ${ww3_res}, ABORT!" exit 1 diff --git a/parm/config/gfs/config.wave b/parm/config/gfs/config.wave index acb4c518ba..74f5b78937 100644 --- a/parm/config/gfs/config.wave +++ b/parm/config/gfs/config.wave @@ -80,7 +80,19 @@ case "${waveGRD}" in export wavepostGRD='glo_500' export waveuoutpGRD=${waveGRD} ;; - *) + "uglo_100km") + #unstructured 100km grid + export waveinterpGRD='glo_200' + export wavepostGRD='' + export waveuoutpGRD=${waveGRD} + ;; + "uglo_m1g16") + #unstructured m1v16 grid + export waveinterpGRD='glo_15mxt' + export wavepostGRD='' + export waveuoutpGRD=${waveGRD} + ;; + *) echo "FATAL ERROR: No grid specific wave config values exist for ${waveGRD}. Aborting." exit 1 ;; diff --git a/parm/wave/at_10m_interp.inp.tmpl b/parm/wave/at_10m_interp.inp.tmpl index b2a80081e1..6f4c1f7099 100755 --- a/parm/wave/at_10m_interp.inp.tmpl +++ b/parm/wave/at_10m_interp.inp.tmpl @@ -5,7 +5,7 @@ $ Start Time DT NSteps $ Total number of grids 2 $ Grid extensions - 'gnh_10m' + 'uglo_m1g16' 'at_10m' $ 0 diff --git a/parm/wave/ep_10m_interp.inp.tmpl b/parm/wave/ep_10m_interp.inp.tmpl index 0848854ccf..23cfd50c2e 100755 --- a/parm/wave/ep_10m_interp.inp.tmpl +++ b/parm/wave/ep_10m_interp.inp.tmpl @@ -5,7 +5,7 @@ $ Start Time DT NSteps $ Total number of grids 2 $ Grid extensions - 'gnh_10m' + 'uglo_m1g16' 'ep_10m' $ 0 diff --git a/parm/wave/glo_15mxt_interp.inp.tmpl b/parm/wave/glo_15mxt_interp.inp.tmpl index 74bc9eebf4..19e9dae684 100755 --- a/parm/wave/glo_15mxt_interp.inp.tmpl +++ b/parm/wave/glo_15mxt_interp.inp.tmpl @@ -3,11 +3,9 @@ $------------------------------------------------ $ Start Time DT NSteps TIME DT NSTEPS $ Total number of grids - 4 + 2 $ Grid extensions - 'gnh_10m' - 'aoc_9km' - 'gsh_15m' + 'uglo_m1g16' 'glo_15mxt' $ 0 diff --git a/parm/wave/glo_200_interp.inp.tmpl b/parm/wave/glo_200_interp.inp.tmpl new file mode 100755 index 0000000000..c238a6fe0b --- /dev/null +++ b/parm/wave/glo_200_interp.inp.tmpl @@ -0,0 +1,12 @@ +$ Input file for interpolation of GLO30m_ext Grid +$------------------------------------------------ +$ Start Time DT NSteps + TIME DT NSTEPS +$ Total number of grids + 2 +$ Grid extensions + 'uglo_100km' + 'glo_200' +$ + 0 +$ diff --git a/parm/wave/glo_30m_interp.inp.tmpl b/parm/wave/glo_30m_interp.inp.tmpl index ea1baf7fc4..c62881202c 100755 --- a/parm/wave/glo_30m_interp.inp.tmpl +++ b/parm/wave/glo_30m_interp.inp.tmpl @@ -3,11 +3,9 @@ $------------------------------------------------ $ Start Time DT NSteps TIME DT NSTEPS $ Total number of grids - 4 + 2 $ Grid extensions - 'gnh_10m' - 'aoc_9km' - 'gsh_15m' + 'uglo_m1g16' 'glo_30m' $ 0 diff --git a/parm/wave/wc_10m_interp.inp.tmpl b/parm/wave/wc_10m_interp.inp.tmpl index abb51b4dfc..8338c91d0c 100755 --- a/parm/wave/wc_10m_interp.inp.tmpl +++ b/parm/wave/wc_10m_interp.inp.tmpl @@ -5,7 +5,7 @@ $ Start Time DT NSteps $ Total number of grids 2 $ Grid extensions - 'gnh_10m' + 'uglo_m1g16' 'wc_10m' $ 0 diff --git a/scripts/exgfs_wave_init.sh b/scripts/exgfs_wave_init.sh index ce903a2284..03fd93bf48 100755 --- a/scripts/exgfs_wave_init.sh +++ b/scripts/exgfs_wave_init.sh @@ -118,6 +118,13 @@ source "${HOMEgfs}/ush/preamble.sh" err=2;export err;${errchk} fi + + if [ -f ${FIXwave}/${grdID}.msh ] + then + cp "${FIXwave}/${grdID}.msh" "${grdID}.msh" + fi + #TO DO: how do we say "it's unstructured, and therefore need to have error check here" + [[ ! -d "${COM_WAVE_PREP}" ]] && mkdir -m 775 -p "${COM_WAVE_PREP}" if [ ${CFP_MP:-"NO"} = "YES" ]; then echo "$nmoddef $USHwave/wave_grid_moddef.sh $grdID > $grdID.out 2>&1" >> cmdfile @@ -166,7 +173,7 @@ source "${HOMEgfs}/ush/preamble.sh" exit=$? fi - if [ "$exit" != '0' ] + if [[ "$exit" != '0' ]] then set +x echo ' ' @@ -195,9 +202,9 @@ source "${HOMEgfs}/ush/preamble.sh" echo '********************************************** ' echo '*** FATAL ERROR : NO MODEL DEFINITION FILE *** ' echo '********************************************** ' - echo " grdID = $grdID" + echo " grdID = ${grdID}" echo ' ' - sed "s/^/$grdID.out : /g" $grdID.out + sed "s/^/${grdID}.out : /g" "${grdID}.out" set_trace err=3;export err;${errchk} fi diff --git a/sorc/build_all.sh b/sorc/build_all.sh index c337374428..dd74c53487 100755 --- a/sorc/build_all.sh +++ b/sorc/build_all.sh @@ -16,7 +16,7 @@ function _usage() { Builds all of the global-workflow components by calling the individual build scripts in sequence. -Usage: ${BASH_SOURCE[0]} [-a UFS_app][-c build_config][-h][-j n][-v] +Usage: ${BASH_SOURCE[0]} [-a UFS_app][-c build_config][-h][-j n][-v][-w] -a UFS_app: Build a specific UFS app instead of the default -g: @@ -29,6 +29,8 @@ Usage: ${BASH_SOURCE[0]} [-a UFS_app][-c build_config][-h][-j n][-v] Build UFS-DA -v: Execute all build scripts with -v option to turn on verbose where supported + -w: + Use unstructured wave grid EOF exit 1 } @@ -40,10 +42,11 @@ _build_ufs_opt="" _build_ufsda="NO" _build_gsi="NO" _verbose_opt="" +_wave_unst="" _build_job_max=20 # Reset option counter in case this script is sourced OPTIND=1 -while getopts ":a:ghj:uv" option; do +while getopts ":a:ghj:uvw" option; do case "${option}" in a) _build_ufs_opt+="-a ${OPTARG} ";; g) _build_gsi="YES" ;; @@ -51,6 +54,7 @@ while getopts ":a:ghj:uv" option; do j) _build_job_max="${OPTARG} ";; u) _build_ufsda="YES" ;; v) _verbose_opt="-v";; + w) _wave_unst="-w";; :) echo "[${BASH_SOURCE[0]}]: ${option} requires an argument" _usage @@ -113,7 +117,7 @@ declare -A build_opts big_jobs=0 build_jobs["ufs"]=8 big_jobs=$((big_jobs+1)) -build_opts["ufs"]="${_verbose_opt} ${_build_ufs_opt}" +build_opts["ufs"]="${_wave_unst} ${_verbose_opt} ${_build_ufs_opt}" build_jobs["upp"]=6 # The UPP is hardcoded to use 6 cores build_opts["upp"]="" @@ -125,11 +129,11 @@ build_jobs["gfs_utils"]=1 build_opts["gfs_utils"]="${_verbose_opt}" build_jobs["ww3prepost"]=3 -build_opts["ww3prepost"]="${_verbose_opt} ${_build_ufs_opt}" +build_opts["ww3prepost"]="${_wave_unst} ${_verbose_opt} ${_build_ufs_opt}" # Optional DA builds if [[ "${_build_ufsda}" == "YES" ]]; then - if [[ "${MACHINE_ID}" != "orion" && "${MACHINE_ID}" != "hera.intel" && "${MACHINE_ID}" != "hercules" ]]; then + if [[ "${MACHINE_ID}" != "orion" && "${MACHINE_ID}" != "hera" && "${MACHINE_ID}" != "hercules" ]]; then echo "NOTE: The GDAS App is not supported on ${MACHINE_ID}. Disabling build." else build_jobs["gdas"]=8 diff --git a/sorc/build_ufs.sh b/sorc/build_ufs.sh index 3e3f879f1a..24ee8c5f13 100755 --- a/sorc/build_ufs.sh +++ b/sorc/build_ufs.sh @@ -7,12 +7,13 @@ cwd=$(pwd) APP="S2SWA" CCPP_SUITES="FV3_GFS_v17_p8_ugwpv1,FV3_GFS_v17_coupled_p8_ugwpv1" # TODO: does the g-w need to build with all these CCPP_SUITES? -while getopts ":da:j:v" option; do +while getopts ":da:j:vw" option; do case "${option}" in d) BUILD_TYPE="DEBUG";; a) APP="${OPTARG}";; j) BUILD_JOBS="${OPTARG}";; v) export BUILD_VERBOSE="YES";; + w) PDLIB="ON";; :) echo "[${BASH_SOURCE[0]}]: ${option} requires an argument" ;; @@ -28,6 +29,7 @@ source "./tests/detect_machine.sh" source "./tests/module-setup.sh" MAKE_OPT="-DAPP=${APP} -D32BIT=ON -DCCPP_SUITES=${CCPP_SUITES}" +[[ ${PDLIB:-"OFF"} = "ON" ]] && MAKE_OPT+=" -DPDLIB=ON" [[ ${BUILD_TYPE:-"Release"} = "DEBUG" ]] && MAKE_OPT+=" -DDEBUG=ON" COMPILE_NR=0 CLEAN_BEFORE=YES diff --git a/sorc/build_ww3prepost.sh b/sorc/build_ww3prepost.sh index 919afaacb3..19cdba98da 100755 --- a/sorc/build_ww3prepost.sh +++ b/sorc/build_ww3prepost.sh @@ -6,12 +6,15 @@ cd "${script_dir}" || exit 1 # Default settings APP="S2SWA" +PDLIB="OFF" -while getopts ":j:a:v" option; do +while getopts ":j:a:dvw" option; do case "${option}" in a) APP="${OPTARG}";; + d) BUILD_TYPE="DEBUG";; j) BUILD_JOBS="${OPTARG}";; v) export BUILD_VERBOSE="YES";; + w) PDLIB="ON";; :) echo "[${BASH_SOURCE[0]}]: ${option} requires an argument" usage @@ -23,14 +26,16 @@ while getopts ":j:a:v" option; do esac done - # Determine which switch to use -if [[ "${APP}" == "ATMW" ]]; then +if [[ "${APP}" == "ATMW" ]]; then ww3switch="model/esmf/switch" -else - ww3switch="model/bin/switch_meshcap" -fi - +else + if [[ "${PDLIB}" == "ON" ]]; then + ww3switch="model/bin/switch_meshcap_pdlib" + else + ww3switch="model/bin/switch_meshcap" + fi +fi # Check final exec folder exists if [[ ! -d "../exec" ]]; then @@ -64,6 +69,8 @@ mkdir -p "${path_build}" || exit 1 cd "${path_build}" || exit 1 echo "Forcing a SHRD build" +buildswitch="${path_build}/switch" + cat "${SWITCHFILE}" > "${path_build}/tempswitch" sed -e "s/DIST/SHRD/g"\ @@ -73,15 +80,21 @@ sed -e "s/DIST/SHRD/g"\ -e "s/MPI / /g"\ -e "s/B4B / /g"\ -e "s/PDLIB / /g"\ + -e "s/SCOTCH / /g"\ + -e "s/METIS / /g"\ -e "s/NOGRB/NCEP2/g"\ "${path_build}/tempswitch" > "${path_build}/switch" rm "${path_build}/tempswitch" -echo "Switch file is ${path_build}/switch with switches:" -cat "${path_build}/switch" +echo "Switch file is ${buildswitch} with switches:" +cat "${buildswitch}" + +#define cmake build options +MAKE_OPT="-DCMAKE_INSTALL_PREFIX=install" +[[ ${BUILD_TYPE:-"Release"} = "DEBUG" ]] && MAKE_OPT+=" -DDEBUG=ON" #Build executables: -cmake "${WW3_DIR}" -DSWITCH="${path_build}/switch" -DCMAKE_INSTALL_PREFIX=install +cmake "${WW3_DIR}" -DSWITCH="${buildswitch}" "${MAKE_OPT}" rc=$? if (( rc != 0 )); then echo "Fatal error in cmake." diff --git a/ush/wave_grid_moddef.sh b/ush/wave_grid_moddef.sh index 5b1b212a16..f704c74af4 100755 --- a/ush/wave_grid_moddef.sh +++ b/ush/wave_grid_moddef.sh @@ -83,8 +83,16 @@ source "$HOMEgfs/ush/preamble.sh" rm -f ww3_grid.inp ln -sf ../ww3_grid.inp.$grdID ww3_grid.inp + + if [ -f ../${grdID}.msh ] + then + rm -f ${grdID}.msh + ln -sf ../${grdID}.msh ${grdID}.msh + fi + + - $EXECwave/ww3_grid 1> grid_${grdID}.out 2>&1 + "${EXECwave}/ww3_grid" 1> "grid_${grdID}.out" 2>&1 err=$? if [ "$err" != '0' ] @@ -99,10 +107,10 @@ source "$HOMEgfs/ush/preamble.sh" exit 3 fi - if [ -f mod_def.ww3 ] + if [[ -f mod_def.ww3 ]] then cp mod_def.ww3 "${COM_WAVE_PREP}/${RUN}wave.mod_def.${grdID}" - mv mod_def.ww3 ../mod_def.$grdID + mv mod_def.ww3 "../mod_def.${grdID}" else set +x echo ' ' @@ -118,6 +126,6 @@ source "$HOMEgfs/ush/preamble.sh" # 3. Clean up cd .. -rm -rf moddef_$grdID +rm -rf "moddef_${grdID}" # End of ww3_mod_def.sh ------------------------------------------------- # diff --git a/versions/fix.ver b/versions/fix.ver index f230188dbb..a2a9caf8e3 100644 --- a/versions/fix.ver +++ b/versions/fix.ver @@ -18,6 +18,6 @@ export mom6_ver=20231219 export orog_ver=20231027 export reg2grb2_ver=20220805 export sfc_climo_ver=20220805 -export ugwd_ver=20220805 +export ugwd_ver=20231027 export verif_ver=20220805 export wave_ver=20240105