Implement global-workflow on AWS

.gitmodules

@@ -1,6 +1,6 @@
 [submodule "sorc/ufs_model.fd"]
   path = sorc/ufs_model.fd
-  url = https://github.com/ufs-community/ufs-weather-model
+  url = https://github.com/ufs-community/ufs-weather-model.git
   ignore = dirty
 [submodule "sorc/wxflow"]
   path = sorc/wxflow

env/AWSPW.env

@@ -14,8 +14,8 @@ fi
 
 step=$1
 
-export launcher="mpiexec.hydra"
-export mpmd_opt=""
+export launcher="srun --mpi=pmi2 -l"
+export mpmd_opt="--distribution=block:block --hint=nomultithread --cpus-per-task=1"
 
 # Configure MPI environment
 export OMP_STACKSIZE=2048000
@@ -36,7 +36,7 @@ if [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then
     (( nnodes = (${!nprocs}+${!ppn}-1)/${!ppn} ))
     (( ntasks = nnodes*${!ppn} ))
     # With ESMF threading, the model wants to use the full node
-    export APRUN_UFS="${launcher} -n ${ntasks}"
+    export APRUN_UFS="${launcher} -n ${ntasks} ${mpmd_opt}"
     unset nprocs ppn nnodes ntasks
 
 elif [[ "${step}" = "post" ]]; then

env/AZUREPW.env

@@ -0,0 +1,298 @@
+#! /usr/bin/env bash
+
+if [[ $# -ne 1 ]]; then
+
+    echo "Must specify an input argument to set runtime environment variables!"
+    echo "argument can be any one of the following:"
+    echo "atmanlrun atmensanlrun aeroanlrun snowanl"
+    echo "anal sfcanl fcst post metp"
+    echo "eobs eupd ecen efcs epos"
+    echo "postsnd awips gempak"
+    exit 1
+
+fi
+
+step=$1
+
+export launcher="srun --mpi=pmi2 -l"
+export mpmd_opt="--distribution=block:block --hint=nomultithread --cpus-per-task=1"
+
+#export POSTAMBLE_CMD='report-mem'
+
+# Configure MPI environment
+#export I_MPI_ADJUST_ALLREDUCE=5
+#export MPI_BUFS_PER_PROC=2048
+#export MPI_BUFS_PER_HOST=2048
+#export MPI_GROUP_MAX=256
+#export MPI_MEMMAP_OFF=1
+#export MP_STDOUTMODE="ORDERED"
+export OMP_STACKSIZE=2048000
+export NTHSTACK=1024000000
+#export LD_BIND_NOW=1
+
+ulimit -s unlimited
+ulimit -a
+
+if [[ "${step}" = "prep" ]] || [[ "${step}" = "prepbufr" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_prep))
+
+    export POE="NO"
+    export BACK="NO"
+    export sys_tp="HERA"
+    export launcher_PREP="srun"
+
+elif [[ "${step}" = "prepsnowobs" ]]; then
+
+    export APRUN_CALCFIMS="${launcher} -n 1"
+
+elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then
+
+    export CFP_MP="YES"
+    if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi
+    export wavempexec=${launcher}
+    export wave_mpmd=${mpmd_opt}
+
+elif [[ "${step}" = "atmanlrun" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_atmanlrun))
+
+    export NTHREADS_ATMANL=${nth_atmanlrun:-${nth_max}}
+    [[ ${NTHREADS_ATMANL} -gt ${nth_max} ]] && export NTHREADS_ATMANL=${nth_max}
+    export APRUN_ATMANL="${launcher} -n ${npe_atmanlrun} --cpus-per-task=${NTHREADS_ATMANL}"
+
+elif [[ "${step}" = "atmensanlrun" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_atmensanlrun))
+
+    export NTHREADS_ATMENSANL=${nth_atmensanlrun:-${nth_max}}
+    [[ ${NTHREADS_ATMENSANL} -gt ${nth_max} ]] && export NTHREADS_ATMENSANL=${nth_max}
+    export APRUN_ATMENSANL="${launcher} -n ${npe_atmensanlrun} --cpus-per-task=${NTHREADS_ATMENSANL}"
+
+elif [[ "${step}" = "aeroanlrun" ]]; then
+
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
+
+    nth_max=$((npe_node_max / npe_node_aeroanlrun))
+
+    export NTHREADS_AEROANL=${nth_aeroanlrun:-${nth_max}}
+    [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max}
+    export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun} --cpus-per-task=${NTHREADS_AEROANL}"
+
+elif [[ "${step}" = "snowanl" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_snowanl))
+
+    export NTHREADS_SNOWANL=${nth_snowanl:-${nth_max}}
+    [[ ${NTHREADS_SNOWANL} -gt ${nth_max} ]] && export NTHREADS_SNOWANL=${nth_max}
+    export APRUN_SNOWANL="${launcher} -n ${npe_snowanl} --cpus-per-task=${NTHREADS_SNOWANL}"
+
+    export APRUN_APPLY_INCR="${launcher} -n 6"
+
+elif [[ "${step}" = "ocnanalbmat" ]]; then
+
+    export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
+
+    export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalbmat}"
+
+elif [[ "${step}" = "ocnanalrun" ]]; then
+
+    export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
+
+    export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalrun}"
+
+elif [[ "${step}" = "ocnanalchkpt" ]]; then
+
+    export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
+
+    export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalchkpt}"
+
+elif [[ "${step}" = "ocnanalecen" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_ocnanalecen))
+
+    export NTHREADS_OCNANALECEN=${nth_ocnanalecen:-${nth_max}}
+    [[ ${NTHREADS_OCNANALECEN} -gt ${nth_max} ]] && export NTHREADS_OCNANALECEN=${nth_max}
+    export APRUN_OCNANALECEN="${launcher} -n ${npe_ocnanalecen} --cpus-per-task=${NTHREADS_OCNANALECEN}"
+
+elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
+
+    export MKL_NUM_THREADS=4
+    export MKL_CBWR=AUTO
+
+    export CFP_MP=${CFP_MP:-"YES"}
+    export USE_CFP=${USE_CFP:-"YES"}
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
+
+    nth_max=$((npe_node_max / npe_node_anal))
+
+    export NTHREADS_GSI=${nth_anal:-${nth_max}}
+    [[ ${NTHREADS_GSI} -gt ${nth_max} ]] && export NTHREADS_GSI=${nth_max}
+    export APRUN_GSI="${launcher} -n ${npe_gsi:-${npe_anal}} --cpus-per-task=${NTHREADS_GSI}"
+
+    export NTHREADS_CALCINC=${nth_calcinc:-1}
+    [[ ${NTHREADS_CALCINC} -gt ${nth_max} ]] && export NTHREADS_CALCINC=${nth_max}
+    export APRUN_CALCINC="${launcher} \$ncmd --cpus-per-task=${NTHREADS_CALCINC}"
+
+    export NTHREADS_CYCLE=${nth_cycle:-12}
+    [[ ${NTHREADS_CYCLE} -gt ${npe_node_max} ]] && export NTHREADS_CYCLE=${npe_node_max}
+    npe_cycle=${ntiles:-6}
+    export APRUN_CYCLE="${launcher} -n ${npe_cycle} --cpus-per-task=${NTHREADS_CYCLE}"
+
+    export NTHREADS_GAUSFCANL=1
+    npe_gausfcanl=${npe_gausfcanl:-1}
+    export APRUN_GAUSFCANL="${launcher} -n ${npe_gausfcanl} --cpus-per-task=${NTHREADS_GAUSFCANL}"
+
+elif [[ "${step}" = "sfcanl" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_sfcanl))
+
+    export NTHREADS_CYCLE=${nth_sfcanl:-14}
+    [[ ${NTHREADS_CYCLE} -gt ${npe_node_max} ]] && export NTHREADS_CYCLE=${npe_node_max}
+    npe_sfcanl=${ntiles:-6}
+    export APRUN_CYCLE="${launcher} -n ${npe_sfcanl} --cpus-per-task=${NTHREADS_CYCLE}"
+
+elif [[ "${step}" = "eobs" ]]; then
+
+    export MKL_NUM_THREADS=4
+    export MKL_CBWR=AUTO
+
+    nth_max=$((npe_node_max / npe_node_eobs))
+
+    export NTHREADS_GSI=${nth_eobs:-${nth_max}}
+    [[ ${NTHREADS_GSI} -gt ${nth_max} ]] && export NTHREADS_GSI=${nth_max}
+    export APRUN_GSI="${launcher} -n ${npe_gsi:-${npe_eobs}} --cpus-per-task=${NTHREADS_GSI}"
+
+    export CFP_MP=${CFP_MP:-"YES"}
+    export USE_CFP=${USE_CFP:-"YES"}
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
+
+elif [[ "${step}" = "eupd" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_eupd))
+
+    export NTHREADS_ENKF=${nth_eupd:-${nth_max}}
+    [[ ${NTHREADS_ENKF} -gt ${nth_max} ]] && export NTHREADS_ENKF=${nth_max}
+    export APRUN_ENKF="${launcher} -n ${npe_enkf:-${npe_eupd}} --cpus-per-task=${NTHREADS_ENKF}"
+
+    export CFP_MP=${CFP_MP:-"YES"}
+    export USE_CFP=${USE_CFP:-"YES"}
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
+
+elif [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then
+
+    if [[ "${CDUMP}" =~ "gfs" ]]; then
+        nprocs="npe_${step}_gfs"
+        ppn="npe_node_${step}_gfs" || ppn="npe_node_${step}"
+    else
+        nprocs="npe_${step}"
+        ppn="npe_node_${step}"
+    fi
+    (( nnodes = (${!nprocs}+${!ppn}-1)/${!ppn} ))
+    (( ntasks = nnodes*${!ppn} ))
+    # With ESMF threading, the model wants to use the full node
+    export APRUN_UFS="${launcher} -n ${ntasks}"
+    unset nprocs ppn nnodes ntasks
+
+elif [[ "${step}" = "upp" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_upp))
+
+    export NTHREADS_UPP=${nth_upp:-1}
+    [[ ${NTHREADS_UPP} -gt ${nth_max} ]] && export NTHREADS_UPP=${nth_max}
+    export APRUN_UPP="${launcher} -n ${npe_upp} --cpus-per-task=${NTHREADS_UPP}"
+
+elif [[ "${step}" = "atmos_products" ]]; then
+
+    export USE_CFP="YES"  # Use MPMD for downstream product generation on Hera
+
+elif [[ "${step}" = "oceanice_products" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_oceanice_products))
+
+    export NTHREADS_OCNICEPOST=${nth_oceanice_products:-1}
+    export APRUN_OCNICEPOST="${launcher} -n 1 --cpus-per-task=${NTHREADS_OCNICEPOST}"
+
+elif [[ "${step}" = "ecen" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_ecen))
+
+    export NTHREADS_ECEN=${nth_ecen:-${nth_max}}
+    [[ ${NTHREADS_ECEN} -gt ${nth_max} ]] && export NTHREADS_ECEN=${nth_max}
+    export APRUN_ECEN="${launcher} -n ${npe_ecen} --cpus-per-task=${NTHREADS_ECEN}"
+
+    export NTHREADS_CHGRES=${nth_chgres:-12}
+    [[ ${NTHREADS_CHGRES} -gt ${npe_node_max} ]] && export NTHREADS_CHGRES=${npe_node_max}
+    export APRUN_CHGRES="time"
+
+    export NTHREADS_CALCINC=${nth_calcinc:-1}
+    [[ ${NTHREADS_CALCINC} -gt ${nth_max} ]] && export NTHREADS_CALCINC=${nth_max}
+    export APRUN_CALCINC="${launcher} -n ${npe_ecen} --cpus-per-task=${NTHREADS_CALCINC}"
+
+elif [[ "${step}" = "esfc" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_esfc))
+
+    export NTHREADS_ESFC=${nth_esfc:-${nth_max}}
+    [[ ${NTHREADS_ESFC} -gt ${nth_max} ]] && export NTHREADS_ESFC=${nth_max}
+    export APRUN_ESFC="${launcher} -n ${npe_esfc} --cpus-per-task=${NTHREADS_ESFC}"
+
+    export NTHREADS_CYCLE=${nth_cycle:-14}
+    [[ ${NTHREADS_CYCLE} -gt ${npe_node_max} ]] && export NTHREADS_CYCLE=${npe_node_max}
+    export APRUN_CYCLE="${launcher} -n ${npe_esfc} --cpus-per-task=${NTHREADS_CYCLE}"
+
+elif [[ "${step}" = "epos" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_epos))
+
+    export NTHREADS_EPOS=${nth_epos:-${nth_max}}
+    [[ ${NTHREADS_EPOS} -gt ${nth_max} ]] && export NTHREADS_EPOS=${nth_max}
+    export APRUN_EPOS="${launcher} -n ${npe_epos} --cpus-per-task=${NTHREADS_EPOS}"
+
+elif [[ "${step}" = "postsnd" ]]; then
+
+    export CFP_MP="YES"
+
+    nth_max=$((npe_node_max / npe_node_postsnd))
+
+    export NTHREADS_POSTSND=${nth_postsnd:-1}
+    [[ ${NTHREADS_POSTSND} -gt ${nth_max} ]] && export NTHREADS_POSTSND=${nth_max}
+    export APRUN_POSTSND="${launcher} -n ${npe_postsnd} --cpus-per-task=${NTHREADS_POSTSND}"
+
+    export NTHREADS_POSTSNDCFP=${nth_postsndcfp:-1}
+    [[ ${NTHREADS_POSTSNDCFP} -gt ${nth_max} ]] && export NTHREADS_POSTSNDCFP=${nth_max}
+    export APRUN_POSTSNDCFP="${launcher} -n ${npe_postsndcfp} ${mpmd_opt}"
+
+elif [[ "${step}" = "awips" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_awips))
+
+    export NTHREADS_AWIPS=${nth_awips:-2}
+    [[ ${NTHREADS_AWIPS} -gt ${nth_max} ]] && export NTHREADS_AWIPS=${nth_max}
+    export APRUN_AWIPSCFP="${launcher} -n ${npe_awips} ${mpmd_opt}"
+
+elif [[ "${step}" = "gempak" ]]; then
+
+    export CFP_MP="YES"
+
+    if [[ ${CDUMP} == "gfs" ]]; then
+        npe_gempak=${npe_gempak_gfs}
+        npe_node_gempak=${npe_node_gempak_gfs}
+    fi
+
+    nth_max=$((npe_node_max / npe_node_gempak))
+
+    export NTHREADS_GEMPAK=${nth_gempak:-1}
+    [[ ${NTHREADS_GEMPAK} -gt ${nth_max} ]] && export NTHREADS_GEMPAK=${nth_max}
+    export APRUN="${launcher} -n ${npe_gempak} ${mpmd_opt}"
+
+
+elif [[ "${step}" = "fit2obs" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_fit2obs))
+
+    export NTHREADS_FIT2OBS=${nth_fit2obs:-1}
+    [[ ${NTHREADS_FIT2OBS} -gt ${nth_max} ]] && export NTHREADS_FIT2OBS=${nth_max}
+    export MPIRUN="${launcher} -n ${npe_fit2obs} --cpus-per-task=${NTHREADS_FIT2OBS}"
+
+fi