Skip to content

Commit

Permalink
Merge remote-tracking branch 'emc/develop' into feature/spack-stack
Browse files Browse the repository at this point in the history
  • Loading branch information
@DavidHuber-NOAA
DavidHuber-NOAA committed Nov 14, 2023
2 parents f86f0d9 + 6f16930 commit b860e45
Show file tree
Hide file tree
Showing 12 changed files with 94 additions and 52 deletions.
8 changes: 4 additions & 4 deletions env/ORION.env
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,20 +132,20 @@ elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then

export NTHREADS_GSI=${nth_anal:-${nth_max}}
[[ ${NTHREADS_GSI} -gt ${nth_max} ]] && export NTHREADS_GSI=${nth_max}
export APRUN_GSI="${launcher} -n ${npe_gsi:-${npe_anal}}"
export APRUN_GSI="${launcher} -n ${npe_gsi:-${npe_anal}} --cpus-per-task=${NTHREADS_GSI}"

export NTHREADS_CALCINC=${nth_calcinc:-1}
[[ ${NTHREADS_CALCINC} -gt ${nth_max} ]] && export NTHREADS_CALCINC=${nth_max}
export APRUN_CALCINC="${launcher} \$ncmd"
export APRUN_CALCINC="${launcher} \$ncmd --cpus-per-task=${NTHREADS_CALCINC}"

export NTHREADS_CYCLE=${nth_cycle:-12}
[[ ${NTHREADS_CYCLE} -gt ${npe_node_max} ]] && export NTHREADS_CYCLE=${npe_node_max}
npe_cycle=${ntiles:-6}
export APRUN_CYCLE="${launcher} -n ${npe_cycle}"
export APRUN_CYCLE="${launcher} -n ${npe_cycle} --cpus-per-task=${NTHREADS_CYCLE}"

export NTHREADS_GAUSFCANL=1
npe_gausfcanl=${npe_gausfcanl:-1}
export APRUN_GAUSFCANL="${launcher} -n ${npe_gausfcanl}"
export APRUN_GAUSFCANL="${launcher} -n ${npe_gausfcanl} --cpus-per-task=${NTHREADS_GAUSFCANL}"

elif [[ "${step}" = "sfcanl" ]]; then
nth_max=$((npe_node_max / npe_node_sfcanl))
Expand Down
23 changes: 20 additions & 3 deletions jobs/rocoto/post.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,26 @@ source "${HOMEgfs}/ush/preamble.sh"
###############################################################

# Source FV3GFS workflow modules
. ${HOMEgfs}/ush/load_fv3gfs_modules.sh
status=$?
[[ ${status} -ne 0 ]] && exit ${status}
# . ${HOMEgfs}/ush/load_fv3gfs_modules.sh
# status=$?
# [[ ${status} -ne 0 ]] && exit ${status}
# Temporarily load modules from UPP
source "${HOMEgfs}/ush/detect_machine.sh"
source "${HOMEgfs}/ush/module-setup.sh"
module use "${HOMEgfs}/sorc/ufs_model.fd/FV3/upp/modulefiles"
module load "${MACHINE_ID}"
module load prod_util
if [[ "${MACHINE_ID}" = "wcoss2" ]]; then
module load cray-pals
module load cfp
else
# shellcheck disable=SC2154
export UTILROOT="${prod_util_ROOT}"
fi
module load grib-util
module load wgrib2
export WGRIB2=wgrib2
# End hack

export job="post"
export jobid="${job}.$$"
Expand Down
2 changes: 1 addition & 1 deletion parm/config/gfs/config.base.emc.dyn
View file
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@ case "${CASE}" in
export OCNRES=025
export waveGRD='glo_025'
;;
"C768")
"C768" | "C1152")
export OCNRES=025
export waveGRD='mx025'
;;
Expand Down
8 changes: 7 additions & 1 deletion parm/config/gfs/config.stage_ic
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,20 @@ case "${CASE}" in
export CPL_ATMIC=GEFS-NoahMP-aerosols-p8c_refactored
export CPL_ICEIC=CPC_refactored
export CPL_OCNIC=CPC3Dvar_refactored
export CPL_WAVIC=GEFSwave20210528v2_refactored
export CPL_WAVIC=workflow_C384_refactored
;;
"C768")
export CPL_ATMIC=HR2_refactored
export CPL_ICEIC=HR1_refactored
export CPL_OCNIC=HR1_refactored
export CPL_WAVIC=HR1_refactored
;;
"C1152")
export CPL_ATMIC=HR2_C1152_refactored
export CPL_ICEIC=HR3_refactored
export CPL_OCNIC=HR3_refactored
export CPL_WAVIC=HR1_refactored
;;
*)
echo "FATAL ERROR Unrecognized resolution: ${CASE}"
exit 1
Expand Down
2 changes: 1 addition & 1 deletion parm/config/gfs/config.ufs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,7 @@ case "${fv3_res}" in
export WRITE_GROUP=4
export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=10 # TODO: refine these numbers when a case is available
export WRITE_GROUP_GFS=4
export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=10 # TODO: refine these numbers when a case is available
export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=20 # TODO: refine these numbers when a case is available
;;
"C3072")
export DELTIM=90
Expand Down
7 changes: 7 additions & 0 deletions sorc/checkout.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,13 @@ for checkout_pid in $(jobs -p); do
wait "${checkout_pid}" || errs=$((errs + $?))
done

# Temporary hack to check out a UPP verison that works on Orion
# This can be removed once the UFS UPP version advances to or beyond 78f369b
cd "${topdir}/ufs_model.fd/FV3/upp" || exit 1
git checkout 78f369b
cd "${topdir}" || exit 1
# End hack

if (( errs > 0 )); then
echo "WARNING: One or more errors encountered during checkout process, please check logs before building"
fi
Expand Down
8 changes: 8 additions & 0 deletions ush/module-setup.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,3 +117,11 @@ elif [[ ${MACHINE_ID} = "noaacloud" ]]; then
else
echo WARNING: UNKNOWN PLATFORM 1>&2
fi

# If this function exists in the environment, run it; else do not
ftype=$(type -t set_strict)
if [[ "${ftype}" == "function" ]]; then
set_strict
else
set +u
fi
5 changes: 4 additions & 1 deletion workflow/applications/gfs_cycled.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ def _get_app_configs(self):
if self.do_ocean:
configs += ['ocnpost']

configs += ['sfcanl', 'analcalc', 'fcst', 'post', 'vrfy', 'fit2obs', 'arch', 'cleanup']
configs += ['sfcanl', 'analcalc', 'fcst', 'post', 'vrfy', 'arch', 'cleanup']

if self.do_hybvar:
if self.do_jediatmens:
Expand All @@ -56,6 +56,9 @@ def _get_app_configs(self):
configs += ['eobs', 'eomg', 'ediag', 'eupd']
configs += ['ecen', 'esfc', 'efcs', 'echgres', 'epos', 'earc']

if self.do_fit2obs:
configs += ['fit2obs']

if self.do_verfozn:
configs += ['verfozn']

Expand Down
2 changes: 1 addition & 1 deletion workflow/hosts/hera.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,4 +22,4 @@ LOCALARCH: 'NO'
ATARDIR: '/NCEPDEV/${HPSS_PROJECT}/1year/${USER}/${machine}/scratch/${PSLOT}'
MAKE_NSSTBUFR: 'NO'
MAKE_ACFTBUFR: 'NO'
SUPPORTED_RESOLUTIONS: ['C768', 'C384', 'C192', 'C96', 'C48']
SUPPORTED_RESOLUTIONS: ['C1152', 'C768', 'C384', 'C192', 'C96', 'C48']
2 changes: 1 addition & 1 deletion workflow/hosts/orion.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,4 +22,4 @@ LOCALARCH: 'NO'
ATARDIR: '${NOSCRUB}/archive_rotdir/${PSLOT}'
MAKE_NSSTBUFR: 'NO'
MAKE_ACFTBUFR: 'NO'
SUPPORTED_RESOLUTIONS: ['C768', 'C384', 'C192', 'C96', 'C48']
SUPPORTED_RESOLUTIONS: ['C1152', 'C768', 'C384', 'C192', 'C96', 'C48']
2 changes: 1 addition & 1 deletion workflow/hosts/wcoss2.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,4 +22,4 @@ LOCALARCH: 'NO'
ATARDIR: '/NCEPDEV/${HPSS_PROJECT}/1year/${USER}/${machine}/scratch/${PSLOT}'
MAKE_NSSTBUFR: 'NO'
MAKE_ACFTBUFR: 'NO'
SUPPORTED_RESOLUTIONS: ['C768', 'C384', 'C192', 'C96', 'C48']
SUPPORTED_RESOLUTIONS: ['C1152', 'C768', 'C384', 'C192', 'C96', 'C48']
77 changes: 39 additions & 38 deletions workflow/rocoto/gfs_tasks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,15 +110,17 @@ def waveinit(self):

resources = self.get_resource('waveinit')
dependencies = None
cycledef = None
if self.app_config.mode in ['cycled']:
deps = []
dep_dict = {'type': 'task', 'name': f'{self.cdump}prep'}
deps.append(rocoto.add_dependency(dep_dict))
dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': '-06:00:00'}
deps.append(rocoto.add_dependency(dep_dict))
if self.cdump in ['gdas']:
dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': '-06:00:00'}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='or', dep=deps)

task = create_wf_task('waveinit', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies)
cycledef = 'gdas_half,gdas' if self.cdump in ['gdas'] else self.cdump
task = create_wf_task('waveinit', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies, cycledef=cycledef)

return task

Expand All @@ -128,9 +130,9 @@ def waveprep(self):
dep_dict = {'type': 'task', 'name': f'{self.cdump}waveinit'}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)

cycledef = 'gdas_half,gdas' if self.cdump in ['gdas'] else self.cdump
resources = self.get_resource('waveprep')
task = create_wf_task('waveprep', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies)
task = create_wf_task('waveprep', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies, cycledef=cycledef)

return task

Expand Down Expand Up @@ -488,6 +490,8 @@ def _fcst_forecast_only(self):
wave_job = 'waveprep' if self.app_config.model_app in ['ATMW'] else 'waveinit'
dep_dict = {'type': 'task', 'name': f'{self.cdump}{wave_job}'}
dependencies.append(rocoto.add_dependency(dep_dict))
dep_dict = {'type': 'task', 'name': f'{self.cdump}waveinit'}
dependencies.append(rocoto.add_dependency(dep_dict))

if self.app_config.do_aero:
# Calculate offset based on CDUMP = gfs | gdas
Expand Down Expand Up @@ -521,10 +525,6 @@ def _fcst_cycled(self):
dep_dict = {'type': 'task', 'name': f'{self.cdump}ocnanalpost'}
dependencies.append(rocoto.add_dependency(dep_dict))

if self.app_config.do_wave and self.cdump in self.app_config.wave_cdumps:
dep_dict = {'type': 'task', 'name': f'{self.cdump}waveprep'}
dependencies.append(rocoto.add_dependency(dep_dict))

if self.app_config.do_aero:
dep_dict = {'type': 'task', 'name': f'{self.cdump}aeroanlfinal'}
dependencies.append(rocoto.add_dependency(dep_dict))
Expand All @@ -540,6 +540,11 @@ def _fcst_cycled(self):
dependencies.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='or', dep=dependencies)

if self.app_config.do_wave and self.cdump in self.app_config.wave_cdumps:
dep_dict = {'type': 'task', 'name': f'{self.cdump}waveprep'}
dependencies.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies)

cycledef = 'gdas_half,gdas' if self.cdump in ['gdas'] else self.cdump

resources = self.get_resource('fcst')
Expand Down Expand Up @@ -917,11 +922,8 @@ def fit2obs(self):
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)

cycledef = 'gdas_half,gdas' if self.cdump in ['gdas'] else self.cdump

resources = self.get_resource('fit2obs')
task = create_wf_task('fit2obs', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies,
cycledef=cycledef)
task = create_wf_task('fit2obs', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies)

return task

Expand Down Expand Up @@ -952,33 +954,32 @@ def metp(self):
def arch(self):
deps = []
dependencies = []
if self.app_config.do_verfozn or self.app_config.do_verfrad or self.app_config.do_vminmon:
if self.app_config.mode in ['cycled']:
if self.cdump in ['gfs']:
if self.app_config.do_vminmon:
dep_dict = {'type': 'task', 'name': f'{self.cdump}vminmon'}
deps.append(rocoto.add_dependency(dep_dict))
elif self.cdump in ['gdas']:
deps2 = []
if self.app_config.do_verfozn:
dep_dict = {'type': 'task', 'name': f'{self.cdump}verfozn'}
deps2.append(rocoto.add_dependency(dep_dict))
if self.app_config.do_verfrad:
dep_dict = {'type': 'task', 'name': f'{self.cdump}verfrad'}
deps2.append(rocoto.add_dependency(dep_dict))
if self.app_config.do_vminmon:
dep_dict = {'type': 'task', 'name': f'{self.cdump}vminmon'}
deps2.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps2)
dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': '-06:00:00'}
dependencies.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='or', dep=dependencies)
if self.app_config.mode in ['cycled']:
if self.cdump in ['gfs']:
if self.app_config.do_vminmon:
dep_dict = {'type': 'task', 'name': f'{self.cdump}vminmon'}
deps.append(rocoto.add_dependency(dep_dict))
elif self.cdump in ['gdas']: # Block for handling half cycle dependencies
deps2 = []
if self.app_config.do_fit2obs:
dep_dict = {'type': 'task', 'name': f'{self.cdump}fit2obs'}
deps2.append(rocoto.add_dependency(dep_dict))
if self.app_config.do_verfozn:
dep_dict = {'type': 'task', 'name': f'{self.cdump}verfozn'}
deps2.append(rocoto.add_dependency(dep_dict))
if self.app_config.do_verfrad:
dep_dict = {'type': 'task', 'name': f'{self.cdump}verfrad'}
deps2.append(rocoto.add_dependency(dep_dict))
if self.app_config.do_vminmon:
dep_dict = {'type': 'task', 'name': f'{self.cdump}vminmon'}
deps2.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps2)
dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': '-06:00:00'}
dependencies.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep_condition='or', dep=dependencies)
if self.app_config.do_vrfy:
dep_dict = {'type': 'task', 'name': f'{self.cdump}vrfy'}
deps.append(rocoto.add_dependency(dep_dict))
if self.cdump in ['gdas'] and self.app_config.do_fit2obs:
dep_dict = {'type': 'task', 'name': f'{self.cdump}fit2obs'}
deps.append(rocoto.add_dependency(dep_dict))
if self.app_config.do_wave:
dep_dict = {'type': 'task', 'name': f'{self.cdump}wavepostsbs'}
deps.append(rocoto.add_dependency(dep_dict))
Expand Down

0 comments on commit b860e45

Please sign in to comment.