Skip to content

Commit

Permalink
Assign machine- and RUN-specific resources (#2672)
Browse files Browse the repository at this point in the history 
Redefine resource variables based explicitly on RUN or CDUMP

Additionally, machine-specific resources are moved out of
config.resources and placed in respective config.resources.{machine}
files.

Resolves #177 #2672
  • Loading branch information
@DavidHuber-NOAA
DavidHuber-NOAA authored Jun 25, 2024
1 parent 4e1b937 commit b902c0b
Show file tree
Hide file tree
Showing 23 changed files with 665 additions and 421 deletions.
32 changes: 5 additions & 27 deletions env/AWSPW.env
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,6 @@
if [[ $# -ne 1 ]]; then

echo "Must specify an input argument to set runtime environment variables!"
echo "argument can be any one of the following:"
echo "atmanlvar atmanlfv3inc atmensanlletkf atmensanlfv3inc aeroanlrun snowanl"
echo "anal sfcanl fcst post metp"
echo "eobs eupd ecen efcs epos"
echo "postsnd awips gempak"
exit 1

fi
Expand All @@ -26,13 +21,11 @@ ulimit -a

if [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then

if [[ "${CDUMP}" =~ "gfs" ]]; then
nprocs="npe_${step}_gfs"
ppn="npe_node_${step}_gfs" || ppn="npe_node_${step}"
else
nprocs="npe_${step}"
ppn="npe_node_${step}"
fi
ppn="npe_node_${step}_${RUN}"
[[ -z "${!ppn+0}" ]] && ppn="npe_node_${step}"
nprocs="npe_${step}_${RUN}"
[[ -z ${!nprocs+0} ]] && nprocs="npe_${step}"

(( nnodes = (${!nprocs}+${!ppn}-1)/${!ppn} ))
(( ntasks = nnodes*${!ppn} ))
# With ESMF threading, the model wants to use the full node
Expand Down Expand Up @@ -109,21 +102,6 @@ elif [[ "${step}" = "awips" ]]; then
[[ ${NTHREADS_AWIPS} -gt ${nth_max} ]] && export NTHREADS_AWIPS=${nth_max}
export APRUN_AWIPSCFP="${launcher} -n ${npe_awips} ${mpmd_opt}"

elif [[ "${step}" = "gempak" ]]; then

export CFP_MP="YES"

if [[ ${CDUMP} == "gfs" ]]; then
npe_gempak=${npe_gempak_gfs}
npe_node_gempak=${npe_node_gempak_gfs}
fi

nth_max=$((npe_node_max / npe_node_gempak))

export NTHREADS_GEMPAK=${nth_gempak:-1}
[[ ${NTHREADS_GEMPAK} -gt ${nth_max} ]] && export NTHREADS_GEMPAK=${nth_max}
export APRUN="${launcher} -n ${npe_gempak} ${mpmd_opt}"


elif [[ "${step}" = "fit2obs" ]]; then

Expand Down
5 changes: 0 additions & 5 deletions env/CONTAINER.env
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,6 @@
if [[ $# -ne 1 ]]; then

echo "Must specify an input argument to set runtime environment variables!"
echo "argument can be any one of the following:"
echo "atmanlvar atmanlfv3inc atmensanlletkf atmensanlfv3inc aeroanlrun snowanl"
echo "anal sfcanl fcst post metp"
echo "eobs eupd ecen efcs epos"
echo "postsnd awips gempak"
exit 1

fi
Expand Down
14 changes: 5 additions & 9 deletions env/GAEA.env
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,6 @@
if [[ $# -ne 1 ]]; then

echo "Must specify an input argument to set runtime environment variables!"
echo "argument can be any one of the following:"
echo "fcst atmos_products"
exit 1

fi
Expand All @@ -19,13 +17,11 @@ ulimit -a

if [[ "${step}" = "fcst" ]]; then

if [[ "${CDUMP}" =~ "gfs" ]]; then
nprocs="npe_${step}_gfs"
ppn="npe_node_${step}_gfs" || ppn="npe_node_${step}"
else
nprocs="npe_${step}"
ppn="npe_node_${step}"
fi
ppn="npe_node_${step}_${RUN}"
[[ -z "${!ppn+0}" ]] && ppn="npe_node_${step}"
nprocs="npe_${step}_${RUN}"
[[ -z ${!nprocs+0} ]] && nprocs="npe_${step}"

(( nnodes = (${!nprocs}+${!ppn}-1)/${!ppn} ))
(( ntasks = nnodes*${!ppn} ))
# With ESMF threading, the model wants to use the full node
Expand Down
21 changes: 5 additions & 16 deletions env/HERA.env
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,6 @@
if [[ $# -ne 1 ]]; then

echo "Must specify an input argument to set runtime environment variables!"
echo "argument can be any one of the following:"
echo "atmanlvar atmanlfv3inc atmensanlletkf atmensanlfv3inc aeroanlrun snowanl"
echo "anal sfcanl fcst post metp"
echo "eobs eupd ecen efcs epos"
echo "postsnd awips gempak"
exit 1

fi
Expand Down Expand Up @@ -219,13 +214,11 @@ elif [[ "${step}" = "eupd" ]]; then

elif [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then

if [[ "${CDUMP}" =~ "gfs" ]]; then
nprocs="npe_${step}_gfs"
ppn="npe_node_${step}_gfs" || ppn="npe_node_${step}"
else
nprocs="npe_${step}"
ppn="npe_node_${step}"
fi
ppn="npe_node_${step}_${RUN}"
[[ -z "${!ppn+0}" ]] && ppn="npe_node_${step}"
nprocs="npe_${step}_${RUN}"
[[ -z ${!nprocs+0} ]] && nprocs="npe_${step}"

(( nnodes = (${!nprocs}+${!ppn}-1)/${!ppn} ))
(( ntasks = nnodes*${!ppn} ))
# With ESMF threading, the model wants to use the full node
Expand Down Expand Up @@ -314,10 +307,6 @@ elif [[ "${step}" = "gempak" ]]; then

export CFP_MP="YES"

if [[ ${CDUMP} == "gfs" ]]; then
npe_node_gempak=${npe_node_gempak_gfs}
fi

nth_max=$((npe_node_max / npe_node_gempak))

export NTHREADS_GEMPAK=${nth_gempak:-1}
Expand Down
29 changes: 7 additions & 22 deletions env/HERCULES.env
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,6 @@
if [[ $# -ne 1 ]]; then

echo "Must specify an input argument to set runtime environment variables!"
echo "argument can be any one of the following:"
echo "fcst post"
echo "Note: Hercules is only set up to run in forecast-only mode"
exit 1

fi
Expand Down Expand Up @@ -211,13 +208,12 @@ case ${step} in
"fcst" | "efcs")

export OMP_STACKSIZE=512M
if [[ "${CDUMP}" =~ "gfs" ]]; then
nprocs="npe_${step}_gfs"
ppn="npe_node_${step}_gfs" || ppn="npe_node_${step}"
else
nprocs="npe_${step}"
ppn="npe_node_${step}"
fi

ppn="npe_node_${step}_${RUN}"
[[ -z "${!ppn+0}" ]] && ppn="npe_node_${step}"
nprocs="npe_${step}_${RUN}"
[[ -z ${!nprocs+0} ]] && nprocs="npe_${step}"

(( nnodes = (${!nprocs}+${!ppn}-1)/${!ppn} ))
(( ntasks = nnodes*${!ppn} ))
# With ESMF threading, the model wants to use the full node
Expand Down Expand Up @@ -312,18 +308,7 @@ case ${step} in
;;
"gempak")

export CFP_MP="YES"

if [[ ${CDUMP} == "gfs" ]]; then
npe_gempak=${npe_gempak_gfs}
npe_node_gempak=${npe_node_gempak_gfs}
fi

nth_max=$((npe_node_max / npe_node_gempak))

export NTHREADS_GEMPAK=${nth_gempak:-1}
[[ ${NTHREADS_GEMPAK} -gt ${nth_max} ]] && export NTHREADS_GEMPAK=${nth_max}
export APRUN="${launcher} -n ${npe_gempak} ${mpmd_opt}"
echo "WARNING: ${step} is not enabled on ${machine}!"

;;
"fit2obs")
Expand Down
17 changes: 5 additions & 12 deletions env/JET.env
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,6 @@
if [[ $# -ne 1 ]]; then

echo "Must specify an input argument to set runtime environment variables!"
echo "argument can be any one of the following:"
echo "atmanlvar atmanlfv3inc atmensanlletkf atmensanlfv3inc aeroanlrun snowanl"
echo "anal sfcanl fcst post metp"
echo "eobs eupd ecen efcs epos"
echo "postsnd awips gempak"
exit 1

fi
Expand Down Expand Up @@ -192,13 +187,11 @@ elif [[ "${step}" = "eupd" ]]; then

elif [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then

if [[ "${CDUMP}" =~ "gfs" ]]; then
nprocs="npe_${step}_gfs"
ppn="npe_node_${step}_gfs" || ppn="npe_node_${step}"
else
nprocs="npe_${step}"
ppn="npe_node_${step}"
fi
ppn="npe_node_${step}_${RUN}"
[[ -z "${!ppn+0}" ]] && ppn="npe_node_${step}"
nprocs="npe_${step}_${RUN}"
[[ -z ${!nprocs+0} ]] && nprocs="npe_${step}"

(( nnodes = (${!nprocs}+${!ppn}-1)/${!ppn} ))
(( ntasks = nnodes*${!ppn} ))
# With ESMF threading, the model wants to use the full node
Expand Down
31 changes: 6 additions & 25 deletions env/ORION.env
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,6 @@
if [[ $# -ne 1 ]]; then

echo "Must specify an input argument to set runtime environment variables!"
echo "argument can be any one of the following:"
echo "atmanlvar atmanlfv3inc atmensanlletkf atmensanlfv3inc aeroanlrun snowanl"
echo "anal sfcanl fcst post metp"
echo "eobs eupd ecen efcs epos"
echo "postsnd awips gempak"
exit 1

fi
Expand Down Expand Up @@ -226,14 +221,11 @@ elif [[ "${step}" = "eupd" ]]; then

elif [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then

export OMP_STACKSIZE=512M
if [[ "${CDUMP}" =~ "gfs" ]]; then
nprocs="npe_${step}_gfs"
ppn="npe_node_${step}_gfs" || ppn="npe_node_${step}"
else
nprocs="npe_${step}"
ppn="npe_node_${step}"
fi
ppn="npe_node_${step}_${RUN}"
[[ -z "${!ppn+0}" ]] && ppn="npe_node_${step}"
nprocs="npe_${step}_${RUN}"
[[ -z ${!nprocs+0} ]] && nprocs="npe_${step}"

(( nnodes = (${!nprocs}+${!ppn}-1)/${!ppn} ))
(( ntasks = nnodes*${!ppn} ))
# With ESMF threading, the model wants to use the full node
Expand Down Expand Up @@ -319,18 +311,7 @@ elif [[ "${step}" = "awips" ]]; then

elif [[ "${step}" = "gempak" ]]; then

export CFP_MP="YES"

if [[ ${CDUMP} == "gfs" ]]; then
npe_gempak=${npe_gempak_gfs}
npe_node_gempak=${npe_node_gempak_gfs}
fi

nth_max=$((npe_node_max / npe_node_gempak))

export NTHREADS_GEMPAK=${nth_gempak:-1}
[[ ${NTHREADS_GEMPAK} -gt ${nth_max} ]] && export NTHREADS_GEMPAK=${nth_max}
export APRUN="${launcher} -n ${npe_gempak} ${mpmd_opt}"
echo "WARNING: ${step} is not enabled on ${machine}!"

elif [[ "${step}" = "fit2obs" ]]; then

Expand Down
18 changes: 6 additions & 12 deletions env/S4.env
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,6 @@
if [[ $# -ne 1 ]]; then

echo "Must specify an input argument to set runtime environment variables!"
echo "argument can be any one of the following:"
echo "atmanlvar atmanlfv3inc atmensanlletkf atmensanlfv3inc aeroanlrun snowanl"
echo "anal sfcanl fcst post metp"
echo "eobs eupd ecen efcs epos"
echo "postsnd awips gempak"
exit 1

fi
Expand Down Expand Up @@ -179,13 +174,11 @@ elif [[ "${step}" = "eupd" ]]; then

elif [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then

if [[ "${CDUMP}" =~ "gfs" ]]; then
nprocs="npe_${step}_gfs"
ppn="npe_node_${step}_gfs" || ppn="npe_node_${step}"
else
nprocs="npe_${step}"
ppn="npe_node_${step}"
fi
ppn="npe_node_${step}_${RUN}"
[[ -z "${!ppn+0}" ]] && ppn="npe_node_${step}"
nprocs="npe_${step}_${RUN}"
[[ -z ${!nprocs+0} ]] && nprocs="npe_${step}"

(( nnodes = (${!nprocs}+${!ppn}-1)/${!ppn} ))
(( ntasks = nnodes*${!ppn} ))
# With ESMF threading, the model wants to use the full node
Expand All @@ -198,6 +191,7 @@ elif [[ "${step}" = "upp" ]]; then

export NTHREADS_UPP=${nth_upp:-1}
[[ ${NTHREADS_UPP} -gt ${nth_max} ]] && export NTHREADS_UPP=${nth_max}
export OMP_NUM_THREADS="${NTHREADS_UPP}"
export APRUN_UPP="${launcher} -n ${npe_upp}"

elif [[ "${step}" = "atmos_products" ]]; then
Expand Down
23 changes: 5 additions & 18 deletions env/WCOSS2.env
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,6 @@
if [[ $# -ne 1 ]]; then

echo "Must specify an input argument to set runtime environment variables!"
echo "argument can be any one of the following:"
echo "atmanlvar atmanlfv3inc atmensanlletkf atmensanlfv3inc aeroanlrun snowanl"
echo "anal sfcanl fcst post metp"
echo "eobs eupd ecen esfc efcs epos"
echo "postsnd awips gempak"
exit 1

fi
Expand Down Expand Up @@ -38,7 +33,6 @@ elif [[ "${step}" = "prep_emissions" ]]; then
elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then

export USE_CFP="YES"
if [[ "${step}" = "waveprep" ]] && [[ "${CDUMP}" = "gfs" ]]; then export NTASKS=${NTASKS_gfs} ; fi
export wavempexec="${launcher} -np"
export wave_mpmd=${mpmd_opt}

Expand Down Expand Up @@ -182,13 +176,11 @@ elif [[ "${step}" = "eupd" ]]; then

elif [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then

if [[ "${CDUMP}" =~ "gfs" ]]; then
nprocs="npe_${step}_gfs"
ppn="npe_node_${step}_gfs" || ppn="npe_node_${step}"
else
nprocs="npe_${step}"
ppn="npe_node_${step}"
fi
ppn="npe_node_${step}_${RUN}"
[[ -z "${!ppn+0}" ]] && ppn="npe_node_${step}"
nprocs="npe_${step}_${RUN}"
[[ -z ${!nprocs+0} ]] && nprocs="npe_${step}"

(( nnodes = (${!nprocs}+${!ppn}-1)/${!ppn} ))
(( ntasks = nnodes*${!ppn} ))
# With ESMF threading, the model wants to use the full node
Expand Down Expand Up @@ -291,11 +283,6 @@ elif [[ "${step}" = "awips" ]]; then

elif [[ "${step}" = "gempak" ]]; then

if [[ ${CDUMP} == "gfs" ]]; then
npe_gempak=${npe_gempak_gfs}
npe_node_gempak=${npe_node_gempak_gfs}
fi

nth_max=$((npe_node_max / npe_node_gempak))

export NTHREADS_GEMPAK=${nth_gempak:-1}
Expand Down
2 changes: 1 addition & 1 deletion modulefiles/module_gwsetup.hercules.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ Load environment to run GFS workflow ci scripts on Hercules
]])

load(pathJoin("contrib","0.1"))
load(pathJoin("rocoto","1.3.5"))
load(pathJoin("rocoto","1.3.7"))

prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/gsi-addon-env/install/modulefiles/Core")

Expand Down
2 changes: 1 addition & 1 deletion parm/config/gefs/config.resources
View file
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,7 @@ case ${step} in
declare -x "wtime_${step}_gfs"="06:00:00"
;;
*)
echo "FATAL ERROR: Resources not defined for job ${job} at resolution ${CASE}"
echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${CASE}"
exit 4
;;
esac
Expand Down
8 changes: 2 additions & 6 deletions parm/config/gfs/config.anal
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,8 @@ if [[ "${CDUMP}" = "gfs" ]] ; then
export DIAG_TARBALL="YES"
fi

export npe_gsi=${npe_anal}

if [[ "${CDUMP}" == "gfs" ]] ; then
export npe_gsi=${npe_anal_gfs}
export nth_anal=${nth_anal_gfs}
fi
npe_var="npe_anal_${RUN/enkf}"
export npe_gsi="${!npe_var}"

# Set parameters specific to L127
if [[ ${LEVS} = "128" ]]; then
Expand Down
Loading

0 comments on commit b902c0b

Please sign in to comment.