Skip to content

Commit

Permalink
Support ATM forecast only on Google (NOAA-EMC#2832)
Browse files Browse the repository at this point in the history
Support global-workflow ATM forecast only runs on Google.

Add/Modify env, yaml, and python scripts changes to make global-workflow
ATM forecast only runs on GSP.

  Resolves NOAA-EMC#2831
  Refs NOAA-EMC#2826
  Refs NOAA-EMC#2711
  • Loading branch information
weihuang-jedi authored Aug 26, 2024
1 parent 1231c9a commit 7a724e0
Show file tree
Hide file tree
Showing 5 changed files with 101 additions and 1 deletion.
55 changes: 55 additions & 0 deletions env/GOOGLEPW.env
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
#! /usr/bin/env bash

if [[ $# -ne 1 ]]; then

echo "Must specify an input argument to set runtime environment variables!"
exit 1

fi

step=$1

export launcher="srun -l --export=ALL"
export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out"

# Configure MPI environment
export OMP_STACKSIZE=2048000
export NTHSTACK=1024000000

ulimit -s unlimited
ulimit -a

# Calculate common variables
# Check first if the dependent variables are set
if [[ -n "${ntasks:-}" && -n "${max_tasks_per_node:-}" && -n "${tasks_per_node:-}" ]]; then
max_threads_per_task=$((max_tasks_per_node / tasks_per_node))
NTHREADSmax=${threads_per_task:-${max_threads_per_task}}
NTHREADS1=${threads_per_task:-1}
[[ ${NTHREADSmax} -gt ${max_threads_per_task} ]] && NTHREADSmax=${max_threads_per_task}
[[ ${NTHREADS1} -gt ${max_threads_per_task} ]] && NTHREADS1=${max_threads_per_task}
APRUN="${launcher} -n ${ntasks}"
else
echo "ERROR config.resources must be sourced before sourcing GOOGLEPW.env"
exit 2
fi

if [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then

export launcher="srun --mpi=pmi2 -l"

(( nnodes = (ntasks+tasks_per_node-1)/tasks_per_node ))
(( ufs_ntasks = nnodes*tasks_per_node ))
# With ESMF threading, the model wants to use the full node
export APRUN_UFS="${launcher} -n ${ufs_ntasks}"
unset nnodes ufs_ntasks

elif [[ "${step}" = "post" ]]; then

export NTHREADS_NP=${NTHREADS1}
export APRUN_NP="${APRUN}"

export NTHREADS_DWN=${threads_per_task_dwn:-1}
[[ ${NTHREADS_DWN} -gt ${max_threads_per_task} ]] && export NTHREADS_DWN=${max_threads_per_task}
export APRUN_DWN="${launcher} -n ${ntasks_dwn}"

fi
8 changes: 8 additions & 0 deletions parm/config/gfs/config.resources
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,14 @@ case ${machine} in
# shellcheck disable=SC2034
mem_node_max=""
;;
"GOOGLEPW")
export PARTITION_BATCH="compute"
npe_node_max=30
max_tasks_per_node=30
# TODO Supply a max mem/node value for GOOGLE
# shellcheck disable=SC2034
mem_node_max=""
;;
"CONTAINER")
max_tasks_per_node=1
# TODO Supply a max mem/node value for a container
Expand Down
11 changes: 11 additions & 0 deletions parm/config/gfs/config.resources.GOOGLEPW
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
#! /usr/bin/env bash

# GOOGLE-specific job resources

export is_exclusive="True"
unset memory

# shellcheck disable=SC2312
for mem_var in $(env | grep '^memory_' | cut -d= -f1); do
unset "${mem_var}"
done
2 changes: 1 addition & 1 deletion workflow/hosts.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ class Host:

SUPPORTED_HOSTS = ['HERA', 'ORION', 'JET', 'HERCULES',
'WCOSS2', 'S4', 'CONTAINER', 'GAEA',
'AWSPW', 'AZUREPW']
'AWSPW', 'AZUREPW', 'GOOGLEPW']

def __init__(self, host=None):

Expand Down
26 changes: 26 additions & 0 deletions workflow/hosts/googlepw.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
BASE_GIT: '' #TODO: This does not yet exist.
DMPDIR: '' # TODO: This does not yet exist.
PACKAGEROOT: '' #TODO: This does not yet exist.
COMINsyn: '' #TODO: This does not yet exist.
HOMEDIR: '/contrib/${USER}'
STMP: '/lustre/${USER}/stmp/'
PTMP: '/lustre/${USER}/ptmp/'
NOSCRUB: '${HOMEDIR}'
ACCOUNT: '${USER}'
SCHEDULER: slurm
QUEUE: batch
QUEUE_SERVICE: batch
PARTITION_BATCH: compute
PARTITION_SERVICE: compute
RESERVATION: ''
CLUSTERS: ''
CHGRP_RSTPROD: 'YES'
CHGRP_CMD: 'chgrp rstprod' # TODO: This is not yet supported.
HPSSARCH: 'NO'
HPSS_PROJECT: emc-global #TODO: See `ATARDIR` below.
BASE_IC: '/bucket/global-workflow-shared-data/ICSDIR/prototype_ICs'
LOCALARCH: 'NO'
ATARDIR: '' # TODO: This will not yet work from GOOGLE.
MAKE_NSSTBUFR: 'NO'
MAKE_ACFTBUFR: 'NO'
SUPPORTED_RESOLUTIONS: ['C48', 'C96'] # TODO: Test and support all cubed-sphere resolutions.

0 comments on commit 7a724e0

Please sign in to comment.