-
Notifications
You must be signed in to change notification settings - Fork 167
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add AWS functionality to workflow setup and forecast job (#1667)
- Loading branch information
1 parent
53a88c3
commit 38cd0bc
Showing
7 changed files
with
227 additions
and
11 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,137 @@ | ||
| #! /usr/bin/env bash | ||
|
|
||
| if [[ $# -ne 1 ]]; then | ||
|
|
||
| echo "Must specify an input argument to set runtime environment variables!" | ||
| echo "argument can be any one of the following:" | ||
| echo "atmanlrun atmensanlrun aeroanlrun landanlrun" | ||
| echo "anal sfcanl fcst post vrfy metp" | ||
| echo "eobs eupd ecen efcs epos" | ||
| echo "postsnd awips gempak" | ||
| exit 1 | ||
|
|
||
| fi | ||
|
|
||
| step=$1 | ||
|
|
||
| export npe_node_max=36 | ||
| export launcher="mpiexec.hydra" | ||
| export mpmd_opt="" | ||
|
|
||
| # Configure MPI environment | ||
| export OMP_STACKSIZE=2048000 | ||
| export NTHSTACK=1024000000 | ||
|
|
||
| ulimit -s unlimited | ||
| ulimit -a | ||
|
|
||
| if [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then | ||
|
|
||
| if [[ "${CDUMP}" =~ "gfs" ]]; then | ||
| nprocs="npe_${step}_gfs" | ||
| ppn="npe_node_${step}_gfs" || ppn="npe_node_${step}" | ||
| else | ||
| nprocs="npe_${step}" | ||
| ppn="npe_node_${step}" | ||
| fi | ||
| (( nnodes = (${!nprocs}+${!ppn}-1)/${!ppn} )) | ||
| (( ntasks = nnodes*${!ppn} )) | ||
| # With ESMF threading, the model wants to use the full node | ||
| export APRUN_UFS="${launcher} -n ${ntasks}" | ||
| unset nprocs ppn nnodes ntasks | ||
|
|
||
| elif [[ "${step}" = "post" ]]; then | ||
|
|
||
| nth_max=$((npe_node_max / npe_node_post)) | ||
|
|
||
| export NTHREADS_NP=${nth_np:-1} | ||
| [[ ${NTHREADS_NP} -gt ${nth_max} ]] && export NTHREADS_NP=${nth_max} | ||
| export APRUN_NP="${launcher} -n ${npe_post}" | ||
|
|
||
| export NTHREADS_DWN=${nth_dwn:-1} | ||
| [[ ${NTHREADS_DWN} -gt ${nth_max} ]] && export NTHREADS_DWN=${nth_max} | ||
| export APRUN_DWN="${launcher} -n ${npe_dwn}" | ||
|
|
||
| elif [[ "${step}" = "ecen" ]]; then | ||
|
|
||
| nth_max=$((npe_node_max / npe_node_ecen)) | ||
|
|
||
| export NTHREADS_ECEN=${nth_ecen:-${nth_max}} | ||
| [[ ${NTHREADS_ECEN} -gt ${nth_max} ]] && export NTHREADS_ECEN=${nth_max} | ||
| export APRUN_ECEN="${launcher} -n ${npe_ecen}" | ||
|
|
||
| export NTHREADS_CHGRES=${nth_chgres:-12} | ||
| [[ ${NTHREADS_CHGRES} -gt ${npe_node_max} ]] && export NTHREADS_CHGRES=${npe_node_max} | ||
| export APRUN_CHGRES="time" | ||
|
|
||
| export NTHREADS_CALCINC=${nth_calcinc:-1} | ||
| [[ ${NTHREADS_CALCINC} -gt ${nth_max} ]] && export NTHREADS_CALCINC=${nth_max} | ||
| export APRUN_CALCINC="${launcher} -n ${npe_ecen}" | ||
|
|
||
| elif [[ "${step}" = "esfc" ]]; then | ||
|
|
||
| nth_max=$((npe_node_max / npe_node_esfc)) | ||
|
|
||
| export NTHREADS_ESFC=${nth_esfc:-${nth_max}} | ||
| [[ ${NTHREADS_ESFC} -gt ${nth_max} ]] && export NTHREADS_ESFC=${nth_max} | ||
| export APRUN_ESFC="${launcher} -n ${npe_esfc}" | ||
|
|
||
| export NTHREADS_CYCLE=${nth_cycle:-14} | ||
| [[ ${NTHREADS_CYCLE} -gt ${npe_node_max} ]] && export NTHREADS_CYCLE=${npe_node_max} | ||
| export APRUN_CYCLE="${launcher} -n ${npe_esfc}" | ||
|
|
||
| elif [[ "${step}" = "epos" ]]; then | ||
|
|
||
| nth_max=$((npe_node_max / npe_node_epos)) | ||
|
|
||
| export NTHREADS_EPOS=${nth_epos:-${nth_max}} | ||
| [[ ${NTHREADS_EPOS} -gt ${nth_max} ]] && export NTHREADS_EPOS=${nth_max} | ||
| export APRUN_EPOS="${launcher} -n ${npe_epos}" | ||
|
|
||
| elif [[ "${step}" = "postsnd" ]]; then | ||
|
|
||
| export CFP_MP="YES" | ||
|
|
||
| nth_max=$((npe_node_max / npe_node_postsnd)) | ||
|
|
||
| export NTHREADS_POSTSND=${nth_postsnd:-1} | ||
| [[ ${NTHREADS_POSTSND} -gt ${nth_max} ]] && export NTHREADS_POSTSND=${nth_max} | ||
| export APRUN_POSTSND="${launcher} -n ${npe_postsnd}" | ||
|
|
||
| export NTHREADS_POSTSNDCFP=${nth_postsndcfp:-1} | ||
| [[ ${NTHREADS_POSTSNDCFP} -gt ${nth_max} ]] && export NTHREADS_POSTSNDCFP=${nth_max} | ||
| export APRUN_POSTSNDCFP="${launcher} -n ${npe_postsndcfp} ${mpmd_opt}" | ||
|
|
||
| elif [[ "${step}" = "awips" ]]; then | ||
|
|
||
| nth_max=$((npe_node_max / npe_node_awips)) | ||
|
|
||
| export NTHREADS_AWIPS=${nth_awips:-2} | ||
| [[ ${NTHREADS_AWIPS} -gt ${nth_max} ]] && export NTHREADS_AWIPS=${nth_max} | ||
| export APRUN_AWIPSCFP="${launcher} -n ${npe_awips} ${mpmd_opt}" | ||
|
|
||
| elif [[ "${step}" = "gempak" ]]; then | ||
|
|
||
| export CFP_MP="YES" | ||
|
|
||
| if [[ ${CDUMP} == "gfs" ]]; then | ||
| npe_gempak=${npe_gempak_gfs} | ||
| npe_node_gempak=${npe_node_gempak_gfs} | ||
| fi | ||
|
|
||
| nth_max=$((npe_node_max / npe_node_gempak)) | ||
|
|
||
| export NTHREADS_GEMPAK=${nth_gempak:-1} | ||
| [[ ${NTHREADS_GEMPAK} -gt ${nth_max} ]] && export NTHREADS_GEMPAK=${nth_max} | ||
| export APRUN="${launcher} -n ${npe_gempak} ${mpmd_opt}" | ||
|
|
||
|
|
||
| elif [[ "${step}" = "fit2obs" ]]; then | ||
|
|
||
| nth_max=$((npe_node_max / npe_node_fit2obs)) | ||
|
|
||
| export NTHREADS_FIT2OBS=${nth_fit2obs:-1} | ||
| [[ ${NTHREADS_FIT2OBS} -gt ${nth_max} ]] && export NTHREADS_FIT2OBS=${nth_max} | ||
| export MPIRUN="${launcher} -n ${npe_fit2obs}" | ||
|
|
||
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,24 @@ | ||
| BASE_GIT: '/scratch1/NCEPDEV/global/glopara/git' #TODO: This does not yet exist. | ||
| DMPDIR: '/scratch1/NCEPDEV/global/glopara/dump' # TODO: This does not yet exist. | ||
| PACKAGEROOT: '/scratch1/NCEPDEV/global/glopara/nwpara' #TODO: This does not yet exist. | ||
| COMROOT: '/scratch1/NCEPDEV/global/glopara/com' #TODO: This does not yet exist. | ||
| COMINsyn: '${COMROOT}/gfs/prod/syndat' #TODO: This does not yet exist. | ||
| HOMEDIR: '/contrib/${USER}' | ||
| STMP: '/lustre/${USER}/stmp2/' | ||
| PTMP: '/lustre/${USER}/stmp4/' | ||
| NOSCRUB: $HOMEDIR | ||
| ACCOUNT: hwufscpldcld | ||
| SCHEDULER: slurm | ||
| QUEUE: batch | ||
| QUEUE_SERVICE: batch | ||
| PARTITION_BATCH: compute | ||
| PARTITION_SERVICE: compute | ||
| CHGRP_RSTPROD: 'YES' | ||
| CHGRP_CMD: 'chgrp rstprod' # TODO: This is not yet supported. | ||
| HPSSARCH: 'YES' | ||
| HPSS_PROJECT: emc-global #TODO: See `ATARDIR` below. | ||
| LOCALARCH: 'NO' | ||
| ATARDIR: '/NCEPDEV/${HPSS_PROJECT}/1year/${USER}/${machine}/scratch/${PSLOT}' # TODO: This will not yet work from AWS. | ||
| MAKE_NSSTBUFR: 'NO' | ||
| MAKE_ACFTBUFR: 'NO' | ||
| SUPPORTED_RESOLUTIONS: ['C48'] # TODO: Test and support all cubed-sphere resolutions. |