From a694cf148a639dac524cce5ad6d93d81aebb8a47 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Thu, 26 Sep 2024 11:05:05 -0400 Subject: [PATCH 1/2] Move the variational scripts to this repository (#2920) This PR "moves and refactors" the variational DA `exscripts` that were in the `GDASapp` to this repository. The ens. var. feature will be replicated/moved in a subsequent PR. Issues have been opened to address reviewer comments at a later time in separate PRs --------- Co-authored-by: Kate.Friedman Co-authored-by: Rahul Mahajan --- ci/cases/gfsv17/ocnanal.yaml | 2 +- env/CONTAINER.env | 4 +- env/HERA.env | 10 +- env/HERCULES.env | 8 +- env/JET.env | 6 +- env/ORION.env | 9 +- jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_CHKPT | 58 --- jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_POST | 49 -- jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_VRFY | 2 +- ...RUN => JGLOBAL_MARINE_ANALYSIS_CHECKPOINT} | 13 +- jobs/JGLOBAL_MARINE_ANALYSIS_FINALIZE | 43 ++ ...REP => JGLOBAL_MARINE_ANALYSIS_INITIALIZE} | 36 +- jobs/JGLOBAL_MARINE_ANALYSIS_VARIATIONAL | 38 ++ jobs/JGLOBAL_MARINE_BMAT | 16 +- .../{ocnanalrun.sh => marineanlchkpt.sh} | 4 +- .../{ocnanalchkpt.sh => marineanlfinal.sh} | 4 +- .../{ocnanalprep.sh => marineanlinit.sh} | 2 +- .../{ocnanalpost.sh => marineanlvar.sh} | 4 +- parm/archive/gdas.yaml.j2 | 8 +- parm/config/gefs/config.base | 3 + parm/config/gfs/config.base | 3 + parm/config/gfs/config.marineanl | 20 + parm/config/gfs/config.marineanlchkpt | 11 + parm/config/gfs/config.marineanlfinal | 10 + parm/config/gfs/config.marineanlinit | 10 + parm/config/gfs/config.marineanlvar | 11 + parm/config/gfs/config.marinebmat | 8 + parm/config/gfs/config.ocnanal | 20 - parm/config/gfs/config.ocnanalchkpt | 11 - parm/config/gfs/config.ocnanalpost | 10 - parm/config/gfs/config.ocnanalprep | 10 - parm/config/gfs/config.ocnanalrun | 11 - parm/config/gfs/config.prepoceanobs | 2 +- parm/config/gfs/config.resources | 10 +- parm/config/gfs/yaml/defaults.yaml | 4 +- .../exglobal_marine_analysis_checkpoint.py | 29 ++ scripts/exglobal_marine_analysis_finalize.py | 27 + .../exglobal_marine_analysis_initialize.py | 24 + .../exglobal_marine_analysis_variational.py | 24 + sorc/gdas.cd | 2 +- sorc/link_workflow.sh | 2 +- ush/python/pygfs/task/marine_analysis.py | 485 ++++++++++++++++++ ush/python/pygfs/task/marine_bmat.py | 78 ++- ush/python/pygfs/utils/marine_da_utils.py | 122 ++++- workflow/applications/gfs_cycled.py | 8 +- workflow/hosts/awspw.yaml | 1 + workflow/hosts/azurepw.yaml | 1 + workflow/hosts/gaea.yaml | 1 + workflow/hosts/googlepw.yaml | 1 + workflow/hosts/hera.yaml | 1 + workflow/hosts/hercules.yaml | 1 + workflow/hosts/jet.yaml | 1 + workflow/hosts/orion.yaml | 1 + workflow/hosts/s4.yaml | 1 + workflow/hosts/wcoss2.yaml | 1 + workflow/rocoto/gfs_tasks.py | 44 +- workflow/rocoto/tasks.py | 2 +- 57 files changed, 993 insertions(+), 334 deletions(-) delete mode 100755 jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_CHKPT delete mode 100755 jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_POST rename jobs/{JGDAS_GLOBAL_OCEAN_ANALYSIS_RUN => JGLOBAL_MARINE_ANALYSIS_CHECKPOINT} (65%) create mode 100755 jobs/JGLOBAL_MARINE_ANALYSIS_FINALIZE rename jobs/{JGDAS_GLOBAL_OCEAN_ANALYSIS_PREP => JGLOBAL_MARINE_ANALYSIS_INITIALIZE} (60%) create mode 100755 jobs/JGLOBAL_MARINE_ANALYSIS_VARIATIONAL rename jobs/rocoto/{ocnanalrun.sh => marineanlchkpt.sh} (82%) rename jobs/rocoto/{ocnanalchkpt.sh => marineanlfinal.sh} (82%) rename jobs/rocoto/{ocnanalprep.sh => marineanlinit.sh} (89%) rename jobs/rocoto/{ocnanalpost.sh => marineanlvar.sh} (82%) create mode 100644 parm/config/gfs/config.marineanl create mode 100644 parm/config/gfs/config.marineanlchkpt create mode 100644 parm/config/gfs/config.marineanlfinal create mode 100644 parm/config/gfs/config.marineanlinit create mode 100644 parm/config/gfs/config.marineanlvar delete mode 100644 parm/config/gfs/config.ocnanal delete mode 100644 parm/config/gfs/config.ocnanalchkpt delete mode 100644 parm/config/gfs/config.ocnanalpost delete mode 100644 parm/config/gfs/config.ocnanalprep delete mode 100644 parm/config/gfs/config.ocnanalrun create mode 100755 scripts/exglobal_marine_analysis_checkpoint.py create mode 100755 scripts/exglobal_marine_analysis_finalize.py create mode 100755 scripts/exglobal_marine_analysis_initialize.py create mode 100755 scripts/exglobal_marine_analysis_variational.py create mode 100644 ush/python/pygfs/task/marine_analysis.py diff --git a/ci/cases/gfsv17/ocnanal.yaml b/ci/cases/gfsv17/ocnanal.yaml index 483250db10..d559f544e4 100644 --- a/ci/cases/gfsv17/ocnanal.yaml +++ b/ci/cases/gfsv17/ocnanal.yaml @@ -16,7 +16,7 @@ base: FHMAX_GFS: 240 ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }} -ocnanal: +marineanl: SOCA_INPUT_FIX_DIR: {{ HOMEgfs }}/fix/gdas/soca/1440x1080x75/soca SOCA_OBS_LIST: {{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml SOCA_NINNER: 100 diff --git a/env/CONTAINER.env b/env/CONTAINER.env index c40543794b..ba01fcf0dd 100755 --- a/env/CONTAINER.env +++ b/env/CONTAINER.env @@ -26,7 +26,7 @@ ulimit -s unlimited ulimit -a -if [ "${step}" = "ocnanalrun" ]; then +if [ "${step}" = "marineanlvar" ]; then export NTHREADS_OCNANAL=1 - export APRUN_OCNANAL="${launcher} -n 2" + export APRUN_MARINEANLVAR="${launcher} -n 2" fi diff --git a/env/HERA.env b/env/HERA.env index 0d77547b5b..09743967b5 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -138,17 +138,15 @@ elif [[ "${step}" = "marinebmat" ]]; then export APRUNCFP="${launcher} -n \$ncmd --multi-prog" export APRUN_MARINEBMAT="${APRUN_default}" -elif [[ "${step}" = "ocnanalrun" ]]; then +elif [[ "${step}" = "marineanlvar" ]]; then export APRUNCFP="${launcher} -n \$ncmd --multi-prog" + export APRUN_MARINEANLVAR="${APRUN_default}" - export APRUN_OCNANAL="${APRUN_default}" - -elif [[ "${step}" = "ocnanalchkpt" ]]; then +elif [[ "${step}" = "marineanlchkpt" ]]; then export APRUNCFP="${launcher} -n \$ncmd --multi-prog" - - export APRUN_OCNANAL="${APRUN_default}" + export APRUN_MARINEANLCHKPT="${APRUN_default}" elif [[ "${step}" = "ocnanalecen" ]]; then diff --git a/env/HERCULES.env b/env/HERCULES.env index 0138e33645..9ec112c699 100755 --- a/env/HERCULES.env +++ b/env/HERCULES.env @@ -133,10 +133,10 @@ case ${step} in export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" export APRUN_MARINEBMAT="${APRUN_default}" ;; - "ocnanalrun") + "marineanlvar") export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" - export APRUN_OCNANAL="${APRUN_default}" + export APRUN_MARINEANLVAR="${APRUN_default}" ;; "ocnanalecen") @@ -148,12 +148,12 @@ case ${step} in [[ ${NTHREADS_OCNANALECEN} -gt ${max_threads_per_task} ]] && export NTHREADS_OCNANALECEN=${max_threads_per_task} export APRUN_OCNANALECEN="${launcher} -n ${ntasks_ocnanalecen} --cpus-per-task=${NTHREADS_OCNANALECEN}" ;; - "ocnanalchkpt") + "marineanlchkpt") export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" export NTHREADS_OCNANAL=${NTHREADSmax} - export APRUN_OCNANAL="${APRUN_default} --cpus-per-task=${NTHREADS_OCNANAL}" + export APRUN_MARINEANLCHKPT="${APRUN_default} --cpus-per-task=${NTHREADS_OCNANAL}" ;; "anal" | "analcalc") diff --git a/env/JET.env b/env/JET.env index f2b018d2d7..dbc249d4d6 100755 --- a/env/JET.env +++ b/env/JET.env @@ -69,7 +69,7 @@ elif [[ "${step}" = "atmensanlsol" ]]; then export NTHREADS_ATMENSANLSOL=${NTHREADSmax} export APRUN_ATMENSANLSOL="${APRUN_default}" - + elif [[ "${step}" = "atmensanlletkf" ]]; then export NTHREADS_ATMENSANLLETKF=${NTHREADSmax} @@ -121,10 +121,10 @@ elif [[ "${step}" = "marinebmat" ]]; then export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" export APRUN_MARINEBMAT="${APRUN_default}" -elif [[ "${step}" = "ocnanalrun" ]]; then +elif [[ "${step}" = "marineanlvar" ]]; then export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" - export APRUN_OCNANAL="${APRUN_default}" + export APRUN_MARINEANLVAR="${APRUN_default}" elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then diff --git a/env/ORION.env b/env/ORION.env index e8c1bcbf58..1bc7eb60d4 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -130,18 +130,19 @@ elif [[ "${step}" = "marinebmat" ]]; then export NTHREADS_MARINEBMAT=${NTHREADSmax} export APRUN_MARINEBMAT="${APRUN_default}" -elif [[ "${step}" = "ocnanalrun" ]]; then +elif [[ "${step}" = "marineanlvar" ]]; then export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" - export APRUN_OCNANAL="${APRUN_default}" + export APRUN_MARINEANLVAR="${APRUN_default}" -elif [[ "${step}" = "ocnanalchkpt" ]]; then +elif [[ "${step}" = "marineanlchkpt" ]]; then export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" export NTHREADS_OCNANAL=${NTHREADSmax} - export APRUN_OCNANAL="${APRUN_default} --cpus-per-task=${NTHREADS_OCNANAL}" + + export APRUN_MARINEANLCHKPT="${APRUN} --cpus-per-task=${NTHREADS_OCNANAL}" elif [[ "${step}" = "ocnanalecen" ]]; then diff --git a/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_CHKPT b/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_CHKPT deleted file mode 100755 index 875fe9d0ee..0000000000 --- a/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_CHKPT +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -source "${HOMEgfs}/ush/preamble.sh" -export WIPE_DATA="NO" -export DATA="${DATAROOT}/${RUN}ocnanal_${cyc}" -source "${HOMEgfs}/ush/jjob_header.sh" -e "ocnanalchkpt" -c "base ocnanal ocnanalchkpt" - - -############################################## -# Set variables used in the script -############################################## -# Ignore possible spelling error (nothing is misspelled) -# shellcheck disable=SC2153 -GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") -export GDATE -export gPDY=${GDATE:0:8} -export gcyc=${GDATE:8:2} -export GDUMP=${GDUMP:-"gdas"} - -export GPREFIX="${GDUMP}.t${gcyc}z." -# Ignore possible spelling error (nothing is misspelled) -# shellcheck disable=SC2153 -export APREFIX="${RUN}.t${cyc}z." - -# Generate COM variables from templates -YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_ATMOS_ANALYSIS - -RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx COM_ATMOS_HISTORY_PREV:COM_ATMOS_HISTORY_TMPL - - -############################################## -# Begin JOB SPECIFIC work -############################################## - -############################################################### -# Run relevant script - -EXSCRIPT=${GDASOCNCHKPTSH:-${HOMEgfs}/sorc/gdas.cd/scripts/exgdas_global_marine_analysis_chkpt.sh} -${EXSCRIPT} -status=$? -[[ ${status} -ne 0 ]] && exit "${status}" - -############################################## -# End JOB SPECIFIC work -############################################## - -############################################## -# Final processing -############################################## -if [[ -e "${pgmout}" ]] ; then - cat "${pgmout}" -fi - -########################################## -# Do not remove the Temporary working directory (do this in POST) -########################################## -cd "${DATAROOT}" || exit 1 - -exit 0 diff --git a/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_POST b/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_POST deleted file mode 100755 index 00597f14f8..0000000000 --- a/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_POST +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash -source "${HOMEgfs}/ush/preamble.sh" -export WIPE_DATA="NO" -export DATA="${DATAROOT}/${RUN}ocnanal_${cyc}" -source "${HOMEgfs}/ush/jjob_header.sh" -e "ocnanalpost" -c "base ocnanalpost" - - -############################################## -# Set variables used in the script -############################################## -# TODO remove this CDUMP declaration when the GDAS script -# exgdas_global_marine_analysis_post.py is updated to look for RUN instead -# of CDUMP. -export CDUMP=${CDUMP:-${RUN:-"gfs"}} -export CDATE=${CDATE:-${PDY}${cyc}} -export GDUMP=${GDUMP:-"gdas"} - -# Generate COM variables from templates -YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_OCEAN_ANALYSIS COM_ICE_ANALYSIS COM_ICE_RESTART - -mkdir -p "${COM_OCEAN_ANALYSIS}" -mkdir -p "${COM_ICE_ANALYSIS}" -mkdir -p "${COM_ICE_RESTART}" - -############################################## -# Begin JOB SPECIFIC work -############################################## - -# Add UFSDA to PYTHONPATH -ufsdaPATH="${HOMEgfs}/sorc/gdas.cd/ush/" -PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${ufsdaPATH}" -export PYTHONPATH - -############################################################### -# Run relevant script -############################################################### - -EXSCRIPT=${GDASOCNPOSTPY:-${HOMEgfs}/sorc/gdas.cd/scripts/exgdas_global_marine_analysis_post.py} -${EXSCRIPT} -status=$? -[[ ${status} -ne 0 ]] && exit "${status}" - -########################################## -# Remove the Temporary working directory -########################################## -cd "${DATAROOT}" || exit 1 -[[ "${KEEPDATA}" = "NO" ]] && rm -rf "${DATA}" - -exit 0 diff --git a/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_VRFY b/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_VRFY index 0d90c46184..31df1e45c7 100755 --- a/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_VRFY +++ b/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_VRFY @@ -1,6 +1,6 @@ #!/bin/bash source "${HOMEgfs}/ush/preamble.sh" -source "${HOMEgfs}/ush/jjob_header.sh" -e "ocnanalprep" -c "base ocnanal ocnanalprep" +source "${HOMEgfs}/ush/jjob_header.sh" -e "marineanlinit" -c "base ocnanal marineanlinit" ############################################## diff --git a/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_RUN b/jobs/JGLOBAL_MARINE_ANALYSIS_CHECKPOINT similarity index 65% rename from jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_RUN rename to jobs/JGLOBAL_MARINE_ANALYSIS_CHECKPOINT index 5871497223..8cd7b1ab7c 100755 --- a/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_RUN +++ b/jobs/JGLOBAL_MARINE_ANALYSIS_CHECKPOINT @@ -1,8 +1,9 @@ #!/bin/bash source "${HOMEgfs}/ush/preamble.sh" export WIPE_DATA="NO" -export DATA="${DATAROOT}/${RUN}ocnanal_${cyc}" -source "${HOMEgfs}/ush/jjob_header.sh" -e "ocnanalrun" -c "base ocnanal ocnanalrun" +export DATAjob="${DATAROOT}/${RUN}marineanalysis.${PDY:-}${cyc}" +export DATA="${DATAjob}/marinevariational" +source "${HOMEgfs}/ush/jjob_header.sh" -e "marineanlchkpt" -c "base marineanl marineanlchkpt" ############################################## @@ -13,11 +14,10 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "ocnanalrun" -c "base ocnanal ocnanalr # Begin JOB SPECIFIC work ############################################## - ############################################################### # Run relevant script -EXSCRIPT=${GDASOCNRUNSH:-${HOMEgfs}/sorc/gdas.cd/scripts/exgdas_global_marine_analysis_run.sh} +EXSCRIPT=${GDASMARINEANALYSIS:-${SCRgfs}/exglobal_marine_analysis_checkpoint.py} ${EXSCRIPT} status=$? [[ ${status} -ne 0 ]] && exit "${status}" @@ -33,9 +33,4 @@ if [[ -e "${pgmout}" ]] ; then cat "${pgmout}" fi -########################################## -# Do not remove the Temporary working directory (do this in POST) -########################################## -cd "${DATAROOT}" || exit 1 - exit 0 diff --git a/jobs/JGLOBAL_MARINE_ANALYSIS_FINALIZE b/jobs/JGLOBAL_MARINE_ANALYSIS_FINALIZE new file mode 100755 index 0000000000..2614639184 --- /dev/null +++ b/jobs/JGLOBAL_MARINE_ANALYSIS_FINALIZE @@ -0,0 +1,43 @@ +#!/bin/bash +source "${HOMEgfs}/ush/preamble.sh" +export WIPE_DATA="NO" +export DATAjob="${DATAROOT}/${RUN}marineanalysis.${PDY:-}${cyc}" +export DATA="${DATAjob}/marinevariational" +source "${HOMEgfs}/ush/jjob_header.sh" -e "marineanlfinal" -c "base marineanl marineanlfinal" + +############################################## +# Set variables used in the script +############################################## + +############################################## +# Begin JOB SPECIFIC work +############################################## + +# Generate COM variables from templates +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COMIN_OBS:COM_OBS_TMPL + +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ + COMOUT_OCEAN_ANALYSIS:COM_OCEAN_ANALYSIS_TMPL \ + COMOUT_ICE_ANALYSIS:COM_ICE_ANALYSIS_TMPL \ + COMOUT_ICE_RESTART:COM_ICE_RESTART_TMPL + +mkdir -p "${COMOUT_OCEAN_ANALYSIS}" +mkdir -p "${COMOUT_ICE_ANALYSIS}" +mkdir -p "${COMOUT_ICE_RESTART}" + +############################################################### +# Run relevant script +############################################################### + +EXSCRIPT=${GDASMARINEANALYSIS:-${SCRgfs}/exglobal_marine_analysis_finalize.py} +${EXSCRIPT} +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +########################################## +# Remove the Temporary working directory +########################################## +cd "${DATAROOT}" || exit 1 +[[ "${KEEPDATA}" = "NO" ]] && rm -rf "${DATA}" + +exit 0 diff --git a/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_PREP b/jobs/JGLOBAL_MARINE_ANALYSIS_INITIALIZE similarity index 60% rename from jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_PREP rename to jobs/JGLOBAL_MARINE_ANALYSIS_INITIALIZE index 664df3aad6..eb167af94d 100755 --- a/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_PREP +++ b/jobs/JGLOBAL_MARINE_ANALYSIS_INITIALIZE @@ -1,7 +1,9 @@ #!/bin/bash + source "${HOMEgfs}/ush/preamble.sh" -export DATA="${DATAROOT}/${RUN}ocnanal_${cyc}" -source "${HOMEgfs}/ush/jjob_header.sh" -e "ocnanalprep" -c "base ocnanal ocnanalprep" +export DATAjob="${DATAROOT}/${RUN}marineanalysis.${PDY:-}${cyc}" +export DATA="${DATAjob}/marinevariational" +source "${HOMEgfs}/ush/jjob_header.sh" -e "marineanlinit" -c "base marineanl marineanlinit" ############################################## @@ -10,42 +12,30 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "ocnanalprep" -c "base ocnanal ocnanal # Ignore possible spelling error (nothing is misspelled) # shellcheck disable=SC2153 GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") -export GDATE -export gPDY=${GDATE:0:8} -export gcyc=${GDATE:8:2} +gPDY=${GDATE:0:8} +gcyc=${GDATE:8:2} export GDUMP=${GDUMP:-"gdas"} -export OPREFIX="${RUN}.t${cyc}z." -export GPREFIX="${GDUMP}.t${gcyc}z." -export APREFIX="${RUN}.t${cyc}z." +############################################## +# Begin JOB SPECIFIC work +############################################## # Generate COM variables from templates YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_OBS RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ - COM_OCEAN_HISTORY_PREV:COM_OCEAN_HISTORY_TMPL \ - COM_ICE_HISTORY_PREV:COM_ICE_HISTORY_TMPL \ - COM_ICE_RESTART_PREV:COM_ICE_RESTART_TMPL + COMIN_OCEAN_HISTORY_PREV:COM_OCEAN_HISTORY_TMPL \ + COMIN_ICE_HISTORY_PREV:COM_ICE_HISTORY_TMPL \ + COMIN_ICE_RESTART_PREV:COM_ICE_RESTART_TMPL YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMIN_OCEAN_BMATRIX:COM_OCEAN_BMATRIX_TMPL \ COMIN_ICE_BMATRIX:COM_ICE_BMATRIX_TMPL -############################################## -# Begin JOB SPECIFIC work -############################################## - -# Add UFSDA to PYTHONPATH -ufsdaPATH="${HOMEgfs}/sorc/gdas.cd/ush/" -# shellcheck disable=SC2311 -pyiodaPATH="${HOMEgfs}/sorc/gdas.cd/build/lib/python$(detect_py_ver)/" -PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${ufsdaPATH}:${pyiodaPATH}" -export PYTHONPATH - ############################################################### # Run relevant script -EXSCRIPT=${GDASOCNPREPPY:-${HOMEgfs}/sorc/gdas.cd/scripts/exgdas_global_marine_analysis_prep.py} +EXSCRIPT=${GDASMARINEANALYSIS:-${SCRgfs}/exglobal_marine_analysis_initialize.py} ${EXSCRIPT} status=$? [[ ${status} -ne 0 ]] && exit "${status}" diff --git a/jobs/JGLOBAL_MARINE_ANALYSIS_VARIATIONAL b/jobs/JGLOBAL_MARINE_ANALYSIS_VARIATIONAL new file mode 100755 index 0000000000..7780353294 --- /dev/null +++ b/jobs/JGLOBAL_MARINE_ANALYSIS_VARIATIONAL @@ -0,0 +1,38 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" +export WIPE_DATA="NO" +export DATAjob="${DATAROOT}/${RUN}marineanalysis.${PDY:-}${cyc}" +export DATA="${DATAjob}/marinevariational" +source "${HOMEgfs}/ush/jjob_header.sh" -e "marineanlvar" -c "base marineanl marineanlvar" + +############################################## +# Set variables used in the script +############################################## + + +############################################## +# Begin JOB SPECIFIC work +############################################## + + +############################################################### +# Run relevant script + +EXSCRIPT=${GDASMARINERUNSH:-${SCRgfs}/exglobal_marine_analysis_variational.py} +${EXSCRIPT} +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +############################################## +# End JOB SPECIFIC work +############################################## + +############################################## +# Final processing +############################################## +if [[ -e "${pgmout}" ]] ; then + cat "${pgmout}" +fi + +exit 0 diff --git a/jobs/JGLOBAL_MARINE_BMAT b/jobs/JGLOBAL_MARINE_BMAT index 3dacec9278..3189df0463 100755 --- a/jobs/JGLOBAL_MARINE_BMAT +++ b/jobs/JGLOBAL_MARINE_BMAT @@ -2,17 +2,17 @@ source "${HOMEgfs}/ush/preamble.sh" -if (( 10#${ENSMEM:-0} > 0 )); then - export DATAjob="${DATAROOT}/${RUN}marinebmat.${PDY:-}${cyc}" - export DATA="${DATAjob}/${jobid}" - # Create the directory to hold ensemble perturbations - export DATAenspert="${DATAjob}/enspert" - if [[ ! -d "${DATAenspert}" ]]; then mkdir -p "${DATAenspert}"; fi -fi +export DATAjob="${DATAROOT}/${RUN}marineanalysis.${PDY:-}${cyc}" +export DATA="${DATAjob}/${jobid}" +# Create the directory to hold ensemble perturbations +export DATAens="${DATAjob}/ensdata" +export DATAstaticb="${DATAjob}/staticb" +if [[ ! -d "${DATAens}" ]]; then mkdir -p "${DATAens}"; fi +if [[ ! -d "${DATAstaticb}" ]]; then mkdir -p "${DATAstaticb}"; fi # source config.base, config.ocnanal and config.marinebmat # and pass marinebmat to ${machine}.env -source "${HOMEgfs}/ush/jjob_header.sh" -e "marinebmat" -c "base ocnanal marinebmat" +source "${HOMEgfs}/ush/jjob_header.sh" -e "marinebmat" -c "base marineanl marinebmat" ############################################## # Set variables used in the script diff --git a/jobs/rocoto/ocnanalrun.sh b/jobs/rocoto/marineanlchkpt.sh similarity index 82% rename from jobs/rocoto/ocnanalrun.sh rename to jobs/rocoto/marineanlchkpt.sh index 5f998af989..69e10a7fa8 100755 --- a/jobs/rocoto/ocnanalrun.sh +++ b/jobs/rocoto/marineanlchkpt.sh @@ -8,11 +8,11 @@ source "${HOMEgfs}/ush/preamble.sh" status=$? [[ ${status} -ne 0 ]] && exit "${status}" -export job="ocnanalrun" +export job="marineanlchkpt" export jobid="${job}.$$" ############################################################### # Execute the JJOB -"${HOMEgfs}"/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_RUN +"${HOMEgfs}"/jobs/JGLOBAL_MARINE_ANALYSIS_CHECKPOINT status=$? exit "${status}" diff --git a/jobs/rocoto/ocnanalchkpt.sh b/jobs/rocoto/marineanlfinal.sh similarity index 82% rename from jobs/rocoto/ocnanalchkpt.sh rename to jobs/rocoto/marineanlfinal.sh index ae98bc8e88..8f0c8fa3a3 100755 --- a/jobs/rocoto/ocnanalchkpt.sh +++ b/jobs/rocoto/marineanlfinal.sh @@ -8,11 +8,11 @@ source "${HOMEgfs}/ush/preamble.sh" status=$? [[ ${status} -ne 0 ]] && exit "${status}" -export job="ocnanalchkpt" +export job="marineanlfinal" export jobid="${job}.$$" ############################################################### # Execute the JJOB -"${HOMEgfs}"/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_CHKPT +"${HOMEgfs}"/jobs/JGLOBAL_MARINE_ANALYSIS_FINALIZE status=$? exit "${status}" diff --git a/jobs/rocoto/ocnanalprep.sh b/jobs/rocoto/marineanlinit.sh similarity index 89% rename from jobs/rocoto/ocnanalprep.sh rename to jobs/rocoto/marineanlinit.sh index 3830fe1c39..953fc0dcfd 100755 --- a/jobs/rocoto/ocnanalprep.sh +++ b/jobs/rocoto/marineanlinit.sh @@ -14,6 +14,6 @@ export jobid="${job}.$$" ############################################################### # Execute the JJOB -"${HOMEgfs}"/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_PREP +"${HOMEgfs}"/jobs/JGLOBAL_MARINE_ANALYSIS_INITIALIZE status=$? exit "${status}" diff --git a/jobs/rocoto/ocnanalpost.sh b/jobs/rocoto/marineanlvar.sh similarity index 82% rename from jobs/rocoto/ocnanalpost.sh rename to jobs/rocoto/marineanlvar.sh index b99a4e05ca..35a21a2bcb 100755 --- a/jobs/rocoto/ocnanalpost.sh +++ b/jobs/rocoto/marineanlvar.sh @@ -8,11 +8,11 @@ source "${HOMEgfs}/ush/preamble.sh" status=$? [[ ${status} -ne 0 ]] && exit "${status}" -export job="ocnanalpost" +export job="marineanlvar" export jobid="${job}.$$" ############################################################### # Execute the JJOB -"${HOMEgfs}"/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_POST +"${HOMEgfs}/jobs/JGLOBAL_MARINE_ANALYSIS_VARIATIONAL" status=$? exit "${status}" diff --git a/parm/archive/gdas.yaml.j2 b/parm/archive/gdas.yaml.j2 index 56e47e595a..030678f31f 100644 --- a/parm/archive/gdas.yaml.j2 +++ b/parm/archive/gdas.yaml.j2 @@ -21,11 +21,11 @@ gdas: - "logs/{{ cycle_YMDH }}/{{ RUN }}atmanlupp.log" {% if DO_JEDIOCNVAR %} - "logs/{{ cycle_YMDH }}/{{ RUN }}prepoceanobs.log" - - "logs/{{ cycle_YMDH }}/{{ RUN }}ocnanalprep.log" + - "logs/{{ cycle_YMDH }}/{{ RUN }}marineanlinit.log" - "logs/{{ cycle_YMDH }}/{{ RUN }}marinebmat.log" - - "logs/{{ cycle_YMDH }}/{{ RUN }}ocnanalrun.log" - - "logs/{{ cycle_YMDH }}/{{ RUN }}ocnanalpost.log" - - "logs/{{ cycle_YMDH }}/{{ RUN }}ocnanalchkpt.log" + - "logs/{{ cycle_YMDH }}/{{ RUN }}marineanlvar.log" + - "logs/{{ cycle_YMDH }}/{{ RUN }}marineanlfinal.log" + - "logs/{{ cycle_YMDH }}/{{ RUN }}marineanlchkpt.log" {% if DOHYBVAR %} - "logs/{{ cycle_YMDH }}/{{ RUN }}ocnanalecen.log" {% endif %} diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base index 883957ed0c..a0bd8b3bd1 100644 --- a/parm/config/gefs/config.base +++ b/parm/config/gefs/config.base @@ -48,6 +48,9 @@ export NOSCRUB="@NOSCRUB@" # Base directories for various builds export BASE_GIT="@BASE_GIT@" +# Base directory for staged data +export BASE_DATA="@BASE_DATA@" + # Toggle to turn on/off GFS downstream processing. export DO_BUFRSND="@DO_BUFRSND@" # BUFR sounding products export DO_GEMPAK="@DO_GEMPAK@" # GEMPAK products diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 27fcbdd055..7fa8245057 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -63,6 +63,9 @@ export NOSCRUB="@NOSCRUB@" # Base directories for various builds export BASE_GIT="@BASE_GIT@" +# Base directory for staged data +export BASE_DATA="@BASE_DATA@" + # Toggle to turn on/off GFS downstream processing. export DO_GOES="@DO_GOES@" # GOES products export DO_BUFRSND="@DO_BUFRSND@" # BUFR sounding products diff --git a/parm/config/gfs/config.marineanl b/parm/config/gfs/config.marineanl new file mode 100644 index 0000000000..a19fc015e2 --- /dev/null +++ b/parm/config/gfs/config.marineanl @@ -0,0 +1,20 @@ +#!/bin/bash + +########## config.marineanl ########## +# configuration common to all ocean analysis tasks + +echo "BEGIN: config.marineanl" + +export MARINE_OBS_YAML_DIR="${PARMgfs}/gdas/soca/obs/config" +export MARINE_OBS_LIST_YAML=@SOCA_OBS_LIST@ +export SOCA_INPUT_FIX_DIR=@SOCA_INPUT_FIX_DIR@ +export SOCA_NINNER=@SOCA_NINNER@ +export DOMAIN_STACK_SIZE=116640000 #TODO: Make the stack size resolution dependent +export SOCA_ENS_BKG_STAGE_YAML_TMPL="${PARMgfs}/gdas/soca/soca_ens_bkg_stage.yaml.j2" +export SOCA_FIX_YAML_TMPL="${PARMgfs}/gdas/soca/soca_fix_stage_${OCNRES}.yaml.j2" +export MARINE_UTILITY_YAML_TMPL="${PARMgfs}/gdas/soca/soca_utils_stage.yaml.j2" +export MARINE_ENSDA_STAGE_BKG_YAML_TMPL="${PARMgfs}/gdas/soca/ensda/stage_ens_mem.yaml.j2" +export MARINE_DET_STAGE_BKG_YAML_TMPL="${PARMgfs}/gdas/soca/soca_det_bkg_stage.yaml.j2" +export MARINE_JCB_GDAS_ALGO="${PARMgfs}/gdas/jcb-gdas/algorithm/marine" + +echo "END: config.marineanl" diff --git a/parm/config/gfs/config.marineanlchkpt b/parm/config/gfs/config.marineanlchkpt new file mode 100644 index 0000000000..7dd6ff79b2 --- /dev/null +++ b/parm/config/gfs/config.marineanlchkpt @@ -0,0 +1,11 @@ +#!/bin/bash + +########## config.marineanlchkpt ########## +# Marine Analysis specific + +echo "BEGIN: config.marineanlchkpt" + +# Get task specific resources +. "${EXPDIR}/config.resources" marineanlchkpt + +echo "END: config.marineanlchkpt" diff --git a/parm/config/gfs/config.marineanlfinal b/parm/config/gfs/config.marineanlfinal new file mode 100644 index 0000000000..ccde289088 --- /dev/null +++ b/parm/config/gfs/config.marineanlfinal @@ -0,0 +1,10 @@ +#!/bin/bash + +########## config.marineanlfinal ########## +# Post Ocn Analysis specific + +echo "BEGIN: config.marineanlfinal" + +# Get task specific resources +. "${EXPDIR}/config.resources" marineanlfinal +echo "END: config.marineanlfinal" diff --git a/parm/config/gfs/config.marineanlinit b/parm/config/gfs/config.marineanlinit new file mode 100644 index 0000000000..01489fc0b8 --- /dev/null +++ b/parm/config/gfs/config.marineanlinit @@ -0,0 +1,10 @@ +#!/bin/bash + +########## config.marineanlinit ########## +# Pre Marine Analysis specific + +echo "BEGIN: config.marineanlinit" + +# Get task specific resources +. "${EXPDIR}/config.resources" marineanlinit +echo "END: config.marineanlinit" diff --git a/parm/config/gfs/config.marineanlvar b/parm/config/gfs/config.marineanlvar new file mode 100644 index 0000000000..5ed6d444eb --- /dev/null +++ b/parm/config/gfs/config.marineanlvar @@ -0,0 +1,11 @@ +#!/bin/bash + +########## config.marineanlvar ########## +# Marine Analysis specific + +echo "BEGIN: config.marineanlvar" + +# Get task specific resources +. "${EXPDIR}/config.resources" marineanlvar + +echo "END: config.marineanlvar" diff --git a/parm/config/gfs/config.marinebmat b/parm/config/gfs/config.marinebmat index d88739dced..00352737d0 100644 --- a/parm/config/gfs/config.marinebmat +++ b/parm/config/gfs/config.marinebmat @@ -8,4 +8,12 @@ echo "BEGIN: config.marinebmat" # Get task specific resources . "${EXPDIR}/config.resources" marinebmat +export BERROR_DIAGB_YAML="${PARMgfs}/gdas/soca/berror/soca_diagb.yaml.j2" +export BERROR_VTSCALES_YAML="${PARMgfs}/gdas/soca/berror/soca_vtscales.yaml.j2" +export BERROR_DIFFV_YAML="${PARMgfs}/gdas/soca/berror/soca_parameters_diffusion_vt.yaml.j2" +export BERROR_HZSCALES_YAML="${PARMgfs}/gdas/soca/berror/soca_setcorscales.yaml" +export BERROR_DIFFH_YAML="${PARMgfs}/gdas/soca/berror/soca_parameters_diffusion_hz.yaml.j2" +export BERROR_ENS_RECENTER_YAML="${PARMgfs}/gdas/soca/berror/soca_ensb.yaml.j2" +export BERROR_HYB_WEIGHTS_YAML="${PARMgfs}/gdas/soca/berror/soca_ensweights.yaml.j2" + echo "END: config.marinebmat" diff --git a/parm/config/gfs/config.ocnanal b/parm/config/gfs/config.ocnanal deleted file mode 100644 index 4d58f2dedf..0000000000 --- a/parm/config/gfs/config.ocnanal +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -########## config.ocnanal ########## -# configuration common to all ocean analysis tasks - -echo "BEGIN: config.ocnanal" - -export OBS_YAML_DIR="${HOMEgfs}/sorc/gdas.cd/parm/soca/obs/config" -export OBS_LIST=@SOCA_OBS_LIST@ # TODO(GA): doesn't look necessary as is to have -export OBS_YAML="${OBS_LIST}" # OBS_LIST and OBS_YAML pick one or add logic -export SOCA_INPUT_FIX_DIR=@SOCA_INPUT_FIX_DIR@ -export SOCA_NINNER=@SOCA_NINNER@ -export DOMAIN_STACK_SIZE=116640000 #TODO: Make the stack size resolution dependent -export SOCA_ENS_BKG_STAGE_YAML_TMPL="${PARMgfs}/gdas/soca/soca_ens_bkg_stage.yaml.j2" -export SOCA_FIX_YAML_TMPL="${PARMgfs}/gdas/soca/soca_fix_stage_${OCNRES}.yaml.j2" - -export JEDI_BIN=${HOMEgfs}/sorc/gdas.cd/build/bin # TODO(GA): remove once analysis "run" - # and "checkpoint" are refactored - -echo "END: config.ocnanal" diff --git a/parm/config/gfs/config.ocnanalchkpt b/parm/config/gfs/config.ocnanalchkpt deleted file mode 100644 index c059fdba42..0000000000 --- a/parm/config/gfs/config.ocnanalchkpt +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -########## config.ocnanalchkpt ########## -# Ocn Analysis specific - -echo "BEGIN: config.ocnanalchkpt" - -# Get task specific resources -. "${EXPDIR}/config.resources" ocnanalchkpt - -echo "END: config.ocnanalchkpt" diff --git a/parm/config/gfs/config.ocnanalpost b/parm/config/gfs/config.ocnanalpost deleted file mode 100644 index bc4d945865..0000000000 --- a/parm/config/gfs/config.ocnanalpost +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -########## config.ocnanalpost ########## -# Post Ocn Analysis specific - -echo "BEGIN: config.ocnanalpost" - -# Get task specific resources -. "${EXPDIR}/config.resources" ocnanalpost -echo "END: config.ocnanalpost" diff --git a/parm/config/gfs/config.ocnanalprep b/parm/config/gfs/config.ocnanalprep deleted file mode 100644 index 225eb089c3..0000000000 --- a/parm/config/gfs/config.ocnanalprep +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -########## config.ocnanalprep ########## -# Pre Ocn Analysis specific - -echo "BEGIN: config.ocnanalprep" - -# Get task specific resources -. "${EXPDIR}/config.resources" ocnanalprep -echo "END: config.ocnanalprep" diff --git a/parm/config/gfs/config.ocnanalrun b/parm/config/gfs/config.ocnanalrun deleted file mode 100644 index 5345b6c684..0000000000 --- a/parm/config/gfs/config.ocnanalrun +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -########## config.ocnanalrun ########## -# Ocn Analysis specific - -echo "BEGIN: config.ocnanalrun" - -# Get task specific resources -. "${EXPDIR}/config.resources" ocnanalrun - -echo "END: config.ocnanalrun" diff --git a/parm/config/gfs/config.prepoceanobs b/parm/config/gfs/config.prepoceanobs index 746ce79257..0963a5c42d 100644 --- a/parm/config/gfs/config.prepoceanobs +++ b/parm/config/gfs/config.prepoceanobs @@ -8,7 +8,7 @@ export OCNOBS2IODAEXEC=${HOMEgfs}/sorc/gdas.cd/build/bin/gdas_obsprovider2ioda.x export SOCA_INPUT_FIX_DIR=@SOCA_INPUT_FIX_DIR@ -export OBS_YAML_DIR="${PARMgfs}/gdas/soca/obs/config" +export MARINE_OBS_YAML_DIR="${PARMgfs}/gdas/soca/obs/config" export OBSPREP_YAML=@OBSPREP_YAML@ export OBS_LIST=@SOCA_OBS_LIST@ export OBS_YAML=${OBS_LIST} diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 0479543ebc..d512f1f885 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -26,7 +26,7 @@ if (( $# != 1 )); then echo "waveinit waveprep wavepostsbs wavepostbndpnt wavepostbndpntbll wavepostpnt" echo "wavegempak waveawipsbulls waveawipsgridded" echo "postsnd awips gempak npoess" - echo "ocnanalprep prepoceanobs marinebmat ocnanalrun ocnanalecen marineanalletkf ocnanalchkpt ocnanalpost ocnanalvrfy" + echo "marineanlinit prepoceanobs marinebmat marineanlvar ocnanalecen marineanalletkf marineanlchkpt marineanlfinal ocnanalvrfy" exit 1 fi @@ -506,7 +506,7 @@ case ${step} in memory="3072M" ;; - "ocnanalprep") + "marineanlinit") walltime="00:10:00" ntasks=1 threads_per_task=1 @@ -541,7 +541,7 @@ case ${step} in tasks_per_node=$(( max_tasks_per_node / threads_per_task )) ;; - "ocnanalrun") + "marineanlvar") ntasks=16 case ${OCNRES} in "025") @@ -632,7 +632,7 @@ case ${step} in ;; - "ocnanalchkpt") + "marineanlchkpt") walltime="00:10:00" ntasks=1 threads_per_task=1 @@ -656,7 +656,7 @@ case ${step} in esac ;; - "ocnanalpost") + "marineanlfinal") walltime="00:30:00" ntasks=${max_tasks_per_node} threads_per_task=1 diff --git a/parm/config/gfs/yaml/defaults.yaml b/parm/config/gfs/yaml/defaults.yaml index 05e1b24012..dfc67d1237 100644 --- a/parm/config/gfs/yaml/defaults.yaml +++ b/parm/config/gfs/yaml/defaults.yaml @@ -52,7 +52,7 @@ snowanl: IO_LAYOUT_X: 1 IO_LAYOUT_Y: 1 -ocnanal: +marineanl: SOCA_INPUT_FIX_DIR: "${FIXgfs}/gdas/soca/72x35x25/soca" SOCA_OBS_LIST: "${PARMgfs}/gdas/soca/obs/obs_list.yaml" # TODO: This is also repeated in oceanprepobs SOCA_NINNER: 100 @@ -61,4 +61,4 @@ prepoceanobs: SOCA_INPUT_FIX_DIR: "${FIXgfs}/gdas/soca/72x35x25/soca" SOCA_OBS_LIST: "${PARMgfs}/gdas/soca/obs/obs_list.yaml" # TODO: This is also repeated in ocnanal OBSPREP_YAML: "${PARMgfs}/gdas/soca/obsprep/obsprep_config.yaml" - DMPDIR: "/scratch1/NCEPDEV/global/glopara/data/experimental_obs" + DMPDIR: "${BASE_DATA}/experimental_obs" diff --git a/scripts/exglobal_marine_analysis_checkpoint.py b/scripts/exglobal_marine_analysis_checkpoint.py new file mode 100755 index 0000000000..84b180b287 --- /dev/null +++ b/scripts/exglobal_marine_analysis_checkpoint.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +# exglobal_marine_analysis_checkpoint.py +# This script creates a MarineAnalysis object +# and runs the checkpoint methods which inserts +# the seaice analysis into the CICE6 restart or +# create a soca MOM6 IAU increment +import os + +from wxflow import Logger, cast_strdict_as_dtypedict +from pygfs.task.marine_analysis import MarineAnalysis + +# Initialize root logger +logger = Logger(level='DEBUG', colored_log=True) + + +if __name__ == '__main__': + + # Take configuration from environment and cast it as python dictionary + config = cast_strdict_as_dtypedict(os.environ) + + # Create a MarineAnalysis object + MarineAnl = MarineAnalysis(config) + + # Prepare the SOCA increment for MOM6 IAU + MarineAnl.checkpoint_mom6_iau('socaincr2mom6.yaml') + + # Insert the seaice analysis into the CICE6 restarts in 2 sequential stages + MarineAnl.checkpoint_cice6('soca_2cice_arctic.yaml') + MarineAnl.checkpoint_cice6('soca_2cice_antarctic.yaml') diff --git a/scripts/exglobal_marine_analysis_finalize.py b/scripts/exglobal_marine_analysis_finalize.py new file mode 100755 index 0000000000..daa3fbb487 --- /dev/null +++ b/scripts/exglobal_marine_analysis_finalize.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +# exglobal_marine_analysis_finalize.py +# This script creates an MarineAnalysis object +# and makes copies of the variational analysis output +# to the COMROOT +import os + +from wxflow import Logger, cast_strdict_as_dtypedict +from pygfs.task.marine_analysis import MarineAnalysis + +# Initialize root logger +logger = Logger(level='DEBUG', colored_log=True) + + +if __name__ == '__main__': + + # Take configuration from environment and cast it as python dictionary + config = cast_strdict_as_dtypedict(os.environ) + + # Create a MarineAnalysis object + MarineAnl = MarineAnalysis(config) + + # Make a copy of the analysis output to the COMROOT + MarineAnl.finalize() + + # Compute the observation space statistics + MarineAnl.obs_space_stats() diff --git a/scripts/exglobal_marine_analysis_initialize.py b/scripts/exglobal_marine_analysis_initialize.py new file mode 100755 index 0000000000..37e45a5b73 --- /dev/null +++ b/scripts/exglobal_marine_analysis_initialize.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +# exglobal_marine_analysis_initialize.py +# This script creates an MarineAnalysis object +# and runs the initialize method +# which create and stage the runtime directory +# and create the YAML configuration +# for a global marine variational analysis +import os + +from wxflow import Logger, cast_strdict_as_dtypedict +from pygfs.task.marine_analysis import MarineAnalysis + +# Initialize root logger +logger = Logger(level='DEBUG', colored_log=True) + + +if __name__ == '__main__': + + # Take configuration from environment and cast it as python dictionary + config = cast_strdict_as_dtypedict(os.environ) + + # Create a MarineAnalysis object + MarineAnl = MarineAnalysis(config) + MarineAnl.initialize() diff --git a/scripts/exglobal_marine_analysis_variational.py b/scripts/exglobal_marine_analysis_variational.py new file mode 100755 index 0000000000..e03c56b1e5 --- /dev/null +++ b/scripts/exglobal_marine_analysis_variational.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +# exglobal_marine_analysis_variational.py +# This script creates an MarineAnalysis object +# and runs the execute method +# which executes the global marine variational analysis +import os + +from wxflow import Logger, cast_strdict_as_dtypedict +from pygfs.task.marine_analysis import MarineAnalysis + +# Initialize root logger +logger = Logger(level='DEBUG', colored_log=True) + + +if __name__ == '__main__': + + # Take configuration from environment and cast it as python dictionary + config = cast_strdict_as_dtypedict(os.environ) + + # Create a MarineAnalysis object + MarineAnl = MarineAnalysis(config) + + # Run the variational application + MarineAnl.variational() diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 7c1c181359..55e895f1dc 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 7c1c181359c2c1952bab3dc1c481bbdb361aa472 +Subproject commit 55e895f1dcf4e6be36eb0eb4c8a7995d429157e0 diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 270a8bb1c9..9426a09ce9 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -275,7 +275,6 @@ if [[ -d "${HOMEgfs}/sorc/gdas.cd/build" ]]; then done fi - #------------------------------ #--add DA Monitor file (NOTE: ensure to use correct version) #------------------------------ @@ -372,6 +371,7 @@ if [[ -d "${HOMEgfs}/sorc/gdas.cd/build" ]]; then "gdas_incr_handler.x" \ "gdas_obsprovider2ioda.x" \ "gdas_socahybridweights.x" \ + "gdassoca_obsstats.x" \ "gdasapp_land_ensrecenter.x" \ "bufr2ioda.x" \ "calcfIMS.exe" \ diff --git a/ush/python/pygfs/task/marine_analysis.py b/ush/python/pygfs/task/marine_analysis.py new file mode 100644 index 0000000000..4e4311b906 --- /dev/null +++ b/ush/python/pygfs/task/marine_analysis.py @@ -0,0 +1,485 @@ +#!/usr/bin/env python3 + +import copy +import os +from logging import getLogger +import pygfs.utils.marine_da_utils as mdau +import glob +import re +import netCDF4 +from multiprocessing import Process +import subprocess +import yaml +from jcb import render + +from wxflow import (AttrDict, + FileHandler, + add_to_datetime, to_timedelta, to_YMD, + parse_j2yaml, + logit, + Executable, + Task, + save_as_yaml, + Template, TemplateConstants, YAMLFile) + +logger = getLogger(__name__.split('.')[-1]) + + +def parse_obs_list_file(obs_list_yaml_path): + # Get the list of observation types from the obs_list.yaml + obs_types = [] + with open(obs_list_yaml_path, 'r') as file: + for line in file: + # Remove leading/trailing whitespace and check if the line is uncommented + line = line.strip() + if line.startswith('- !INC') and not line.startswith('#'): + # Extract the type using regex + match = re.search(r'\$\{MARINE_OBS_YAML_DIR\}/(.+)\.yaml', line) + if match: + obs_types.append(str(match.group(1))) + return obs_types + + +class MarineAnalysis(Task): + """ + Class for global marine analysis tasks + """ + @logit(logger, name="MarineAnalysis") + def __init__(self, config): + super().__init__(config) + _calc_scale_exec = os.path.join(self.task_config.HOMEgfs, 'ush', 'soca', 'calc_scales.py') + _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config.assim_freq}H") / 2) + _window_end = add_to_datetime(self.task_config.current_cycle, to_timedelta(f"{self.task_config.assim_freq}H") / 2) + + # compute the relative path from self.task_config.DATA to self.task_config.DATAenspert + if self.task_config.NMEM_ENS > 0: + _enspert_relpath = os.path.relpath(self.task_config.DATAenspert, self.task_config.DATA) + else: + _enspert_relpath = None + + # Create a local dictionary that is repeatedly used across this class + local_dict = AttrDict( + { + 'PARMsoca': os.path.join(self.task_config.PARMgfs, 'gdas', 'soca'), + 'MARINE_WINDOW_BEGIN': _window_begin, + 'MARINE_WINDOW_BEGIN_ISO': _window_begin.strftime('%Y-%m-%dT%H:%M:%SZ'), + 'MARINE_WINDOW_END': _window_end, + 'MARINE_WINDOW_LENGTH': f"PT{self.task_config['assim_freq']}H", + 'MARINE_WINDOW_MIDDLE': self.task_config.current_cycle, + 'MARINE_WINDOW_MIDDLE_ISO': self.task_config.current_cycle.strftime('%Y-%m-%dT%H:%M:%SZ'), + 'ENSPERT_RELPATH': _enspert_relpath, + 'CALC_SCALE_EXEC': _calc_scale_exec, + 'OPREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z." + } + ) + + # Extend task_config with local_dict + self.task_config.update(local_dict) + + @logit(logger) + def initialize(self: Task) -> None: + """Initialize the marine analysis + + This method will initialize the marine analysis. + This includes: + - staging the deterministic backgrounds (middle of window) + - staging SOCA fix files + - staging static ensemble members (optional) + - staging ensemble members (optional) + - generating the YAML files for the JEDI and GDASApp executables + - creating output directories + """ + super().initialize() + + # prepare the directory structure to run SOCA + self._prep_scratch_dir() + + # fetch observations from COMROOT + # TODO(G.V. or A.E.): Keep a copy of the obs in the scratch fs after the obs prep job + self._fetch_observations() + + # stage the ocean and ice backgrounds for FGAT + bkg_list = parse_j2yaml(self.task_config.MARINE_DET_STAGE_BKG_YAML_TMPL, self.task_config) + FileHandler(bkg_list).sync() + + # stage the soca grid + FileHandler({'copy': [[os.path.join(self.task_config.COMIN_OCEAN_BMATRIX, 'soca_gridspec.nc'), + os.path.join(self.task_config.DATA, 'soca_gridspec.nc')]]}).sync() + + # link the flow dependent static B resources from the B-matrix task of the same cycle + os.symlink('../staticb', 'staticb') + + # hybrid EnVAR case + if self.task_config.DOHYBVAR == "YES" or self.task_config.NMEM_ENS > 2: + # stage ensemble membersfiles for use in hybrid background error + logger.debug(f"Stage ensemble members for the hybrid background error") + mdau.stage_ens_mem(self.task_config) + + # prepare the yaml configuration to run the SOCA variational application + self._prep_variational_yaml() + + # prepare the yaml configuration to run the SOCA to MOM6 IAU increment + self._prep_checkpoint() + + @logit(logger) + def _fetch_observations(self: Task) -> None: + """Fetch observations from COMIN_OBS + + This method will fetch the observations for the cycle and check the + list against what is available for the cycle. + """ + + # get the list of observations + obs_list_config = YAMLFile(self.task_config.MARINE_OBS_LIST_YAML) + obs_list_config = Template.substitute_structure(obs_list_config, TemplateConstants.DOLLAR_PARENTHESES, self.task_config) + obs_list_config = {'observations': obs_list_config} + logger.info(f"{obs_list_config}") + + obs_files = [] + for ob in obs_list_config['observations']['observers']: + logger.info(f"******** {self.task_config.OPREFIX}{ob['obs space']['name'].lower()}.{to_YMD(self.task_config.PDY)}{self.task_config.cyc}.nc4") + obs_files.append(f"{self.task_config.OPREFIX}{ob['obs space']['name'].lower()}.{to_YMD(self.task_config.PDY)}{self.task_config.cyc}.nc4") + obs_list = [] + + # copy obs from COM_OBS to DATA/obs + for obs_file in obs_files: + logger.info(f"******* {obs_file}") + obs_src = os.path.join(self.task_config.COM_OBS, obs_file) + obs_dst = os.path.join(self.task_config.DATA, 'obs', obs_file) + logger.info(f"******* {obs_src}") + if os.path.exists(obs_src): + logger.info(f"******* fetching {obs_file}") + obs_list.append([obs_src, obs_dst]) + else: + logger.info(f"******* {obs_file} is not in the database") + + FileHandler({'copy': obs_list}).sync() + + @logit(logger) + def _prep_scratch_dir(self: Task) -> None: + """Create and stage all the resources needed to run SOCA/JEDI, including the necesssary + directory structure to run the SOCA variational application + """ + logger.info(f"---------------- Setup runtime environement") + + anl_dir = self.task_config.DATA + + # create analysis directories + diags = os.path.join(anl_dir, 'diags') # output dir for soca DA obs space + obs_in = os.path.join(anl_dir, 'obs') # input " " + anl_out = os.path.join(anl_dir, 'Data') # output dir for soca DA + FileHandler({'mkdir': [diags, obs_in, anl_out]}).sync() + + # stage fix files + logger.info(f"Staging SOCA fix files from {self.task_config.SOCA_INPUT_FIX_DIR}") + soca_fix_list = parse_j2yaml(self.task_config.SOCA_FIX_YAML_TMPL, self.task_config) + FileHandler(soca_fix_list).sync() + + # prepare the MOM6 input.nml + mdau.prep_input_nml(self.task_config) + + # stage the soca utility yamls (gridgen, fields and ufo mapping yamls) + logger.info(f"Staging SOCA utility yaml files from {self.task_config.PARMsoca}") + soca_utility_list = parse_j2yaml(self.task_config.MARINE_UTILITY_YAML_TMPL, self.task_config) + FileHandler(soca_utility_list).sync() + + @logit(logger) + def _prep_variational_yaml(self: Task) -> None: + """Create the yaml configuration to run the SOCA variational application + """ + + # prepare background list for the pseudo model, check bkg date for consistency + mdau.gen_bkg_list(bkg_path='./bkg', + window_begin=self.task_config.MARINE_WINDOW_BEGIN, + yaml_name='bkg_list.yaml') + + # Make a copy of the env config before modifying to avoid breaking something else + envconfig_jcb = copy.deepcopy(self.task_config) + logger.info(f"---------------- Prepare the yaml configuration") + logger.info(f"{envconfig_jcb}") # Prepare the yaml configuration + + # Add the things to the envconfig in order to template JCB files + envconfig_jcb['PARMgfs'] = self.task_config.PARMgfs + envconfig_jcb['nmem_ens'] = self.task_config.NMEM_ENS + envconfig_jcb['berror_model'] = 'marine_background_error_static_diffusion' + if self.task_config.NMEM_ENS > 3: + envconfig_jcb['berror_model'] = 'marine_background_error_hybrid_diffusion_diffusion' + envconfig_jcb['DATA'] = self.task_config.DATA + envconfig_jcb['OPREFIX'] = self.task_config.OPREFIX + envconfig_jcb['PDY'] = os.getenv('PDY') + envconfig_jcb['cyc'] = os.getenv('cyc') + envconfig_jcb['SOCA_NINNER'] = self.task_config.SOCA_NINNER + envconfig_jcb['obs_list'] = ['adt_rads_all'] + + # Write obs_list_short + save_as_yaml(parse_obs_list_file(self.task_config.MARINE_OBS_LIST_YAML), 'obs_list_short.yaml') + os.environ['OBS_LIST_SHORT'] = 'obs_list_short.yaml' + + # Render the JCB configuration files + jcb_base_yaml = os.path.join(self.task_config.PARMsoca, 'marine-jcb-base.yaml') + jcb_algo_yaml = os.path.join(self.task_config.PARMsoca, 'marine-jcb-3dfgat.yaml.j2') + + jcb_base_config = YAMLFile(path=jcb_base_yaml) + jcb_base_config = Template.substitute_structure(jcb_base_config, TemplateConstants.DOUBLE_CURLY_BRACES, envconfig_jcb.get) + jcb_base_config = Template.substitute_structure(jcb_base_config, TemplateConstants.DOLLAR_PARENTHESES, envconfig_jcb.get) + jcb_algo_config = YAMLFile(path=jcb_algo_yaml) + jcb_algo_config = Template.substitute_structure(jcb_algo_config, TemplateConstants.DOUBLE_CURLY_BRACES, envconfig_jcb.get) + jcb_algo_config = Template.substitute_structure(jcb_algo_config, TemplateConstants.DOLLAR_PARENTHESES, envconfig_jcb.get) + + # Override base with the application specific config + jcb_config = {**jcb_base_config, **jcb_algo_config} + + # convert datetime to string + jcb_config['window_begin'] = self.task_config.MARINE_WINDOW_BEGIN.strftime('%Y-%m-%dT%H:%M:%SZ') + jcb_config['window_middle'] = self.task_config.MARINE_WINDOW_MIDDLE.strftime('%Y-%m-%dT%H:%M:%SZ') + + # Render the full JEDI configuration file using JCB + jedi_config = render(jcb_config) + + # Save the JEDI configuration file + var_yaml_jcb = 'var.yaml' + mdau.clean_empty_obsspaces(jedi_config, target=var_yaml_jcb, app='var') + + def _prep_checkpoint(self: Task) -> None: + """Create the yaml configuration to run the SOCA to MOM6 IAU increment + """ + # prepare the socaincr2mom6.yaml + logger.info("Generate the SOCA to MOM6 IAU increment YAML file") + data = {'marine_window_begin': self.task_config.MARINE_WINDOW_BEGIN_ISO, + 'marine_window_middle': self.task_config.MARINE_WINDOW_MIDDLE_ISO} + soca2mom6inc_config = parse_j2yaml(path=os.path.join(self.task_config.MARINE_JCB_GDAS_ALGO, 'socaincr2mom6.yaml.j2'), + data=data) + soca2mom6inc_config.save(os.path.join(self.task_config.DATA, 'socaincr2mom6.yaml')) + + # prepare the SOCA to CICE YAML file + logger.info("Generate the SOCA to CICE RST YAML file") + + # set the restart date, dependent on the cycling type + if self.task_config.DOIAU: + # forecast initialized at the begining of the DA window + fcst_begin = self.task_config.MARINE_WINDOW_BEGIN_ISO + rst_date = self.task_config.MARINE_WINDOW_BEGIN.strftime('%Y%m%d.%H%M%S') + else: + # forecast initialized at the middle of the DA window + fcst_begin = self.task_config.MARINE_WINDOW_MIDDLE_ISO + rst_date = self.task_config.MARINE_WINDOW_MIDDLE.strftime('%Y%m%d.%H%M%S') + + # make a copy of the CICE6 restart + ice_rst = os.path.join(self.task_config.COMIN_ICE_RESTART_PREV, f'{rst_date}.cice_model.res.nc') + ice_rst_ana = os.path.join(self.task_config.DATA, 'Data', rst_date + '.cice_model.res.nc') + FileHandler({'copy': [[ice_rst, ice_rst_ana]]}).sync() + + # prepare the necessary configuration for the SOCA to CICE application + soca2cice_param = AttrDict({ + "ocn_ana": f"./Data/ocn.3dvarfgat_pseudo.an.{self.task_config.MARINE_WINDOW_MIDDLE_ISO}.nc", + "ice_ana": f"./Data/ice.3dvarfgat_pseudo.an.{self.task_config.MARINE_WINDOW_MIDDLE_ISO}.nc", + "ice_rst": ice_rst_ana, + "fcst_begin": fcst_begin + }) + logger.debug(f"{soca2cice_param}") + + # render the SOCA to CICE YAML file for the Arctic and Antarctic + logger.info("render the SOCA to CICE YAML file for the Arctic and Antarctic") + varchgyamls = ['soca_2cice_arctic.yaml', 'soca_2cice_antarctic.yaml'] + for varchgyaml in varchgyamls: + soca2cice_config = parse_j2yaml(path=os.path.join(self.task_config.MARINE_JCB_GDAS_ALGO, f'{varchgyaml}.j2'), + data=soca2cice_param) + soca2cice_config.save(os.path.join(self.task_config.DATA, varchgyaml)) + + @logit(logger) + def variational(self: Task) -> None: + # link gdas_soca_gridgen.x + mdau.link_executable(self.task_config, 'gdas.x') + exec_cmd = Executable(self.task_config.APRUN_MARINEANLVAR) + exec_name = os.path.join(self.task_config.DATA, 'gdas.x') + exec_cmd.add_default_arg(exec_name) + exec_cmd.add_default_arg('soca') + exec_cmd.add_default_arg('variational') + exec_cmd.add_default_arg('var.yaml') + + mdau.run(exec_cmd) + + @logit(logger) + def checkpoint_cice6(self: Task, soca2ciceyaml) -> None: + # link gdas_soca_gridgen.x + mdau.link_executable(self.task_config, 'gdas.x') + exec_cmd = Executable(self.task_config.APRUN_MARINEANLCHKPT) + exec_name = os.path.join(self.task_config.DATA, 'gdas.x') + exec_cmd.add_default_arg(exec_name) + exec_cmd.add_default_arg('soca') + exec_cmd.add_default_arg('convertstate') + exec_cmd.add_default_arg(soca2ciceyaml) + + mdau.run(exec_cmd) + + @logit(logger) + def checkpoint_mom6_iau(self: Task, socaincr2mom6yaml) -> None: + # link gdas_incr_handler.x + mdau.link_executable(self.task_config, 'gdas_incr_handler.x') + exec_cmd = Executable(self.task_config.APRUN_MARINEANLCHKPT) + exec_name = os.path.join(self.task_config.DATA, 'gdas_incr_handler.x') + exec_cmd.add_default_arg(exec_name) + exec_cmd.add_default_arg(socaincr2mom6yaml) + + mdau.run(exec_cmd) + + @logit(logger) + def finalize(self: Task) -> None: + """Finalize the marine analysis job + This method saves the results of the deterministic variational analysis to the COMROOT + """ + + def list_all_files(dir_in, dir_out, wc='*', fh_list=[]): + files = glob.glob(os.path.join(dir_in, wc)) + for file_src in files: + file_dst = os.path.join(dir_out, os.path.basename(file_src)) + fh_list.append([file_src, file_dst]) + return fh_list + + # variables of convenience + com_ocean_analysis = self.task_config.COMOUT_OCEAN_ANALYSIS + com_ice_analysis = self.task_config.COMOUT_ICE_ANALYSIS + com_ice_restart = self.task_config.COMOUT_ICE_RESTART + anl_dir = self.task_config.DATA + cdate = self.task_config.CDATE + pdy = self.task_config.PDY + staticsoca_dir = self.task_config.SOCA_INPUT_FIX_DIR + RUN = self.task_config.RUN + cyc = str(self.task_config.cyc).zfill(2) + bcyc = str(self.task_config.MARINE_WINDOW_BEGIN.hour).zfill(2) + bdate = self.task_config.MARINE_WINDOW_BEGIN_ISO + mdate = self.task_config.MARINE_WINDOW_MIDDLE_ISO + nmem_ens = int(self.task_config.NMEM_ENS) + + logger.info(f"---------------- Copy from RUNDIR to COMOUT") + + post_file_list = [] + + # Make a copy the IAU increment + post_file_list.append([os.path.join(anl_dir, 'inc.nc'), + os.path.join(com_ocean_analysis, f'{RUN}.t{cyc}z.ocninc.nc')]) + + domains = ['ocn', 'ice'] + for domain in domains: + ''' + # Copy of the diagonal of the background error for the cycle + post_file_list.append([os.path.join(anl_dir, f'{domain}.bkgerr_stddev.incr.{mdate}.nc'), + os.path.join(com_ocean_analysis, f'{RUN}.t{cyc}z.{domain}.bkgerr_stddev.nc')]) + + # Copy the recentering error + if nmem_ens > 2: + post_file_list.append([os.path.join(anl_dir, 'static_ens', f'{domain}.ssh_recentering_error.incr.{bdate}.nc'), + os.path.join(com_ocean_analysis, f'{RUN}.t{cyc}z.{domain}.recentering_error.nc')]) + ''' + + # Copy the ice and ocean increments + post_file_list.append([os.path.join(anl_dir, 'Data', f'{domain}.3dvarfgat_pseudo.incr.{mdate}.nc'), + os.path.join(com_ocean_analysis, f'{RUN}.t{cyc}z.{domain}.incr.nc')]) + + # Copy the analysis at the start of the window + post_file_list.append([os.path.join(anl_dir, 'Data', f'{domain}.3dvarfgat_pseudo.an.{mdate}.nc'), + os.path.join(com_ocean_analysis, f'{RUN}.t{cyc}z.{domain}ana.nc')]) + + # Copy of the ssh diagnostics + ''' + if nmem_ens > 2: + for string in ['ssh_steric_stddev', 'ssh_unbal_stddev', 'ssh_total_stddev', 'steric_explained_variance']: + post_file_list.append([os.path.join(anl_dir, 'static_ens', f'ocn.{string}.incr.{bdate}.nc'), + os.path.join(com_ocean_analysis, f'{RUN}.t{cyc}z.ocn.{string}.nc')]) + ''' + + # Copy DA grid (computed for the start of the window) + post_file_list.append([os.path.join(anl_dir, 'soca_gridspec.nc'), + os.path.join(com_ocean_analysis, f'{RUN}.t{bcyc}z.ocngrid.nc')]) + + # Copy the CICE analysis restart + if os.getenv('DOIAU') == "YES": + cice_rst_date = self.task_config.MARINE_WINDOW_BEGIN.strftime('%Y%m%d.%H%M%S') + else: + cice_rst_date = cdate.strftime('%Y%m%d.%H%M%S') + + post_file_list.append([os.path.join(anl_dir, 'Data', f'{cice_rst_date}.cice_model.res.nc'), + os.path.join(com_ice_analysis, f'{cice_rst_date}.cice_model_anl.res.nc')]) + + FileHandler({'copy': post_file_list}).sync() + + # create COM sub-directories + FileHandler({'mkdir': [os.path.join(com_ocean_analysis, 'diags'), + os.path.join(com_ocean_analysis, 'bump'), + os.path.join(com_ocean_analysis, 'yaml')]}).sync() + + # ioda output files + fh_list = list_all_files(os.path.join(anl_dir, 'diags'), + os.path.join(com_ocean_analysis, 'diags')) + + # yaml configurations + fh_list = list_all_files(os.path.join(anl_dir), + os.path.join(com_ocean_analysis, 'yaml'), wc='*.yaml', fh_list=fh_list) + + FileHandler({'copy': fh_list}).sync() + + @logit(logger) + def obs_space_stats(self: Task) -> None: + """Observation space statistics + This method computes a few basic statistics on the observation spaces + """ + + # obs space statistics + logger.info(f"---------------- Compute basic stats") + diags_list = glob.glob(os.path.join(os.path.join(self.task_config.COMOUT_OCEAN_ANALYSIS, 'diags', '*.nc4'))) + obsstats_j2yaml = str(os.path.join(self.task_config.PARMgfs, 'gdas', 'soca', 'obs', 'obs_stats.yaml.j2')) + + # function to create a minimalist ioda obs sapce + def create_obs_space(data): + os_dict = {"obs space": { + "name": data["obs_space"], + "obsdatain": { + "engine": {"type": "H5File", "obsfile": data["obsfile"]} + }, + "simulated variables": [data["variable"]] + }, + "variable": data["variable"], + "experiment identifier": data["pslot"], + "csv output": data["csv_output"] + } + return os_dict + + # get the experiment id + pslot = self.task_config.PSLOT + + # iterate through the obs spaces and generate the yaml for gdassoca_obsstats.x + obs_spaces = [] + for obsfile in diags_list: + + # define an obs space name + obs_space = re.sub(r'\.\d{10}\.nc4$', '', os.path.basename(obsfile)) + + # get the variable name, assume 1 variable per file + nc = netCDF4.Dataset(obsfile, 'r') + variable = next(iter(nc.groups["ObsValue"].variables)) + nc.close() + + # filling values for the templated yaml + data = {'obs_space': os.path.basename(obsfile), + 'obsfile': obsfile, + 'pslot': pslot, + 'variable': variable, + 'csv_output': os.path.join(self.task_config.COMOUT_OCEAN_ANALYSIS, + f"{self.task_config.OPREFIX}ocn.{obs_space}.stats.csv")} + obs_spaces.append(create_obs_space(data)) + + # create the yaml + data = {'obs_spaces': obs_spaces} + conf = parse_j2yaml(path=obsstats_j2yaml, data=data) + stats_yaml = 'diag_stats.yaml' + conf.save(stats_yaml) + + # run the application + mdau.link_executable(self.task_config, 'gdassoca_obsstats.x') + command = f"{os.getenv('launcher')} -n 1" + exec_cmd = Executable(command) + exec_name = os.path.join(self.task_config.DATA, 'gdassoca_obsstats.x') + exec_cmd.add_default_arg(exec_name) + exec_cmd.add_default_arg(stats_yaml) + + mdau.run(exec_cmd) diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index 4770583934..93329f05ac 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -26,29 +26,24 @@ def __init__(self, config): super().__init__(config) _home_gdas = os.path.join(self.task_config.HOMEgfs, 'sorc', 'gdas.cd') _calc_scale_exec = os.path.join(self.task_config.HOMEgfs, 'ush', 'soca', 'calc_scales.py') - _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config.assim_freq}H") / 2) - _window_end = add_to_datetime(self.task_config.current_cycle, to_timedelta(f"{self.task_config.assim_freq}H") / 2) + _window_begin = add_to_datetime(self.task_config.current_cycle, + -to_timedelta(f"{self.task_config.assim_freq}H") / 2) + _window_end = add_to_datetime(self.task_config.current_cycle, + to_timedelta(f"{self.task_config.assim_freq}H") / 2) # compute the relative path from self.task_config.DATA to self.task_config.DATAenspert - if self.task_config.NMEM_ENS > 0: - _enspert_relpath = os.path.relpath(self.task_config.DATAenspert, self.task_config.DATA) - else: - _enspert_relpath = None + _enspert_relpath = os.path.relpath(self.task_config.DATAens, self.task_config.DATA) # Create a local dictionary that is repeatedly used across this class local_dict = AttrDict( { - 'HOMEgdas': _home_gdas, + 'PARMsoca': os.path.join(self.task_config.PARMgfs, 'gdas', 'soca'), 'MARINE_WINDOW_BEGIN': _window_begin, 'MARINE_WINDOW_END': _window_end, 'MARINE_WINDOW_MIDDLE': self.task_config.current_cycle, - 'BERROR_YAML_DIR': os.path.join(_home_gdas, 'parm', 'soca', 'berror'), - 'UTILITY_YAML_TMPL': os.path.join(_home_gdas, 'parm', 'soca', 'soca_utils_stage.yaml.j2'), - 'MARINE_ENSDA_STAGE_BKG_YAML_TMPL': os.path.join(_home_gdas, 'parm', 'soca', 'ensda', 'stage_ens_mem.yaml.j2'), - 'MARINE_DET_STAGE_BKG_YAML_TMPL': os.path.join(_home_gdas, 'parm', 'soca', 'soca_det_bkg_stage.yaml.j2'), 'ENSPERT_RELPATH': _enspert_relpath, 'CALC_SCALE_EXEC': _calc_scale_exec, - 'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", + 'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z." } ) @@ -61,7 +56,7 @@ def initialize(self: Task) -> None: This method will initialize a global B-Matrix. This includes: - - staging the deterministic backgrounds (middle of window) + - staging the deterministic backgrounds - staging SOCA fix files - staging static ensemble members (optional) - staging ensemble members (optional) @@ -84,39 +79,35 @@ def initialize(self: Task) -> None: FileHandler(bkg_list).sync() # stage the soca utility yamls (gridgen, fields and ufo mapping yamls) - logger.info(f"Staging SOCA utility yaml files from {self.task_config.HOMEgfs}/parm/gdas/soca") - soca_utility_list = parse_j2yaml(self.task_config.UTILITY_YAML_TMPL, self.task_config) + logger.info(f"Staging SOCA utility yaml files") + soca_utility_list = parse_j2yaml(self.task_config.MARINE_UTILITY_YAML_TMPL, self.task_config) FileHandler(soca_utility_list).sync() # generate the variance partitioning YAML file - logger.debug("Generate variance partitioning YAML file") - diagb_config = parse_j2yaml(path=os.path.join(self.task_config.BERROR_YAML_DIR, 'soca_diagb.yaml.j2'), - data=self.task_config) + logger.info(f"Generate variance partitioning YAML file from {self.task_config.BERROR_DIAGB_YAML}") + diagb_config = parse_j2yaml(path=self.task_config.BERROR_DIAGB_YAML, data=self.task_config) diagb_config.save(os.path.join(self.task_config.DATA, 'soca_diagb.yaml')) # generate the vertical decorrelation scale YAML file - logger.debug("Generate the vertical correlation scale YAML file") - vtscales_config = parse_j2yaml(path=os.path.join(self.task_config.BERROR_YAML_DIR, 'soca_vtscales.yaml.j2'), - data=self.task_config) + logger.info(f"Generate the vertical correlation scale YAML file from {self.task_config.BERROR_VTSCALES_YAML}") + vtscales_config = parse_j2yaml(path=self.task_config.BERROR_VTSCALES_YAML, data=self.task_config) vtscales_config.save(os.path.join(self.task_config.DATA, 'soca_vtscales.yaml')) # generate vertical diffusion scale YAML file - logger.debug("Generate vertical diffusion YAML file") - diffvz_config = parse_j2yaml(path=os.path.join(self.task_config.BERROR_YAML_DIR, 'soca_parameters_diffusion_vt.yaml.j2'), - data=self.task_config) + logger.info(f"Generate vertical diffusion YAML file from {self.task_config.BERROR_DIFFV_YAML}") + diffvz_config = parse_j2yaml(path=self.task_config.BERROR_DIFFV_YAML, data=self.task_config) diffvz_config.save(os.path.join(self.task_config.DATA, 'soca_parameters_diffusion_vt.yaml')) # generate the horizontal diffusion YAML files if True: # TODO(G): skip this section once we have optimized the scales # stage the correlation scale configuration - logger.debug("Generate correlation scale YAML file") - FileHandler({'copy': [[os.path.join(self.task_config.BERROR_YAML_DIR, 'soca_setcorscales.yaml'), + logger.info(f"Generate correlation scale YAML file from {self.task_config.BERROR_HZSCALES_YAML}") + FileHandler({'copy': [[self.task_config.BERROR_HZSCALES_YAML, os.path.join(self.task_config.DATA, 'soca_setcorscales.yaml')]]}).sync() # generate horizontal diffusion scale YAML file - logger.debug("Generate horizontal diffusion scale YAML file") - diffhz_config = parse_j2yaml(path=os.path.join(self.task_config.BERROR_YAML_DIR, 'soca_parameters_diffusion_hz.yaml.j2'), - data=self.task_config) + logger.info(f"Generate horizontal diffusion scale YAML file from {self.task_config.BERROR_DIFFH_YAML}") + diffhz_config = parse_j2yaml(path=self.task_config.BERROR_DIFFH_YAML, data=self.task_config) diffhz_config.save(os.path.join(self.task_config.DATA, 'soca_parameters_diffusion_hz.yaml')) # hybrid EnVAR case @@ -127,19 +118,20 @@ def initialize(self: Task) -> None: # generate ensemble recentering/rebalancing YAML file logger.debug("Generate ensemble recentering YAML file") - ensrecenter_config = parse_j2yaml(path=os.path.join(self.task_config.BERROR_YAML_DIR, 'soca_ensb.yaml.j2'), - data=self.task_config) + ensrecenter_config = parse_j2yaml(path=self.task_config.BERROR_ENS_RECENTER_YAML, data=self.task_config) ensrecenter_config.save(os.path.join(self.task_config.DATA, 'soca_ensb.yaml')) # generate ensemble weights YAML file - logger.debug("Generate ensemble recentering YAML file: {self.task_config.abcd_yaml}") - hybridweights_config = parse_j2yaml(path=os.path.join(self.task_config.BERROR_YAML_DIR, 'soca_ensweights.yaml.j2'), - data=self.task_config) + logger.debug("Generate hybrid-weigths YAML file") + hybridweights_config = parse_j2yaml(path=self.task_config.BERROR_HYB_WEIGHTS_YAML, data=self.task_config) hybridweights_config.save(os.path.join(self.task_config.DATA, 'soca_ensweights.yaml')) - # need output dir for ensemble perturbations and static B-matrix - logger.debug("Create empty diagb directories to receive output from executables") - FileHandler({'mkdir': [os.path.join(self.task_config.DATA, 'diagb')]}).sync() + # create the symbolic link to the static B-matrix directory + link_target = os.path.join(self.task_config.DATAstaticb) + link_name = os.path.join(self.task_config.DATA, 'staticb') + if os.path.exists(link_name): + os.remove(link_name) + os.symlink(link_target, link_name) @logit(logger) def gridgen(self: Task) -> None: @@ -290,12 +282,12 @@ def finalize(self: Task) -> None: logger.info(f"Copying the diffusion coefficient files to the ROTDIR") diffusion_coeff_list = [] for diff_type in ['hz', 'vt']: - src = os.path.join(self.task_config.DATA, f"{diff_type}_ocean.nc") + src = os.path.join(self.task_config.DATAstaticb, f"{diff_type}_ocean.nc") dest = os.path.join(self.task_config.COMOUT_OCEAN_BMATRIX, f"{self.task_config.APREFIX}{diff_type}_ocean.nc") diffusion_coeff_list.append([src, dest]) - src = os.path.join(self.task_config.DATA, f"hz_ice.nc") + src = os.path.join(self.task_config.DATAstaticb, f"hz_ice.nc") dest = os.path.join(self.task_config.COMOUT_ICE_BMATRIX, f"{self.task_config.APREFIX}hz_ice.nc") diffusion_coeff_list.append([src, dest]) @@ -308,13 +300,17 @@ def finalize(self: Task) -> None: window_end_iso = self.task_config.MARINE_WINDOW_END.strftime('%Y-%m-%dT%H:%M:%SZ') # ocean diag B - src = os.path.join(self.task_config.DATA, 'diagb', f"ocn.bkgerr_stddev.incr.{window_end_iso}.nc") + os.rename(os.path.join(self.task_config.DATAstaticb, f"ocn.bkgerr_stddev.incr.{window_end_iso}.nc"), + os.path.join(self.task_config.DATAstaticb, f"ocn.bkgerr_stddev.nc")) + src = os.path.join(self.task_config.DATAstaticb, f"ocn.bkgerr_stddev.nc") dst = os.path.join(self.task_config.COMOUT_OCEAN_BMATRIX, f"{self.task_config.APREFIX}ocean.bkgerr_stddev.nc") diagb_list.append([src, dst]) # ice diag B - src = os.path.join(self.task_config.DATA, 'diagb', f"ice.bkgerr_stddev.incr.{window_end_iso}.nc") + os.rename(os.path.join(self.task_config.DATAstaticb, f"ice.bkgerr_stddev.incr.{window_end_iso}.nc"), + os.path.join(self.task_config.DATAstaticb, f"ice.bkgerr_stddev.nc")) + src = os.path.join(self.task_config.DATAstaticb, f"ice.bkgerr_stddev.nc") dst = os.path.join(self.task_config.COMOUT_ICE_BMATRIX, f"{self.task_config.APREFIX}ice.bkgerr_stddev.nc") diagb_list.append([src, dst]) diff --git a/ush/python/pygfs/utils/marine_da_utils.py b/ush/python/pygfs/utils/marine_da_utils.py index 2be76ac028..e1b2ac2d4d 100644 --- a/ush/python/pygfs/utils/marine_da_utils.py +++ b/ush/python/pygfs/utils/marine_da_utils.py @@ -1,7 +1,9 @@ -import f90nml +from datetime import datetime, timedelta +import dateutil.parser as dparser import os +from netCDF4 import Dataset from logging import getLogger -import xarray as xr +import yaml from wxflow import (FileHandler, logit, @@ -9,6 +11,7 @@ AttrDict, parse_j2yaml, Executable, + save_as_yaml, jinja) logger = getLogger(__name__.split('.')[-1]) @@ -23,9 +26,9 @@ def run(exec_cmd: Executable) -> None: logger.debug(f"Executing {exec_cmd}") exec_cmd() except OSError: - raise OSError(f"Failed to execute {exec_cmd}") + raise OSError(f"FATAL ERROR: Failed to execute {exec_cmd}") except Exception: - raise WorkflowException(f"An error occured during execution of {exec_cmd}") + raise WorkflowException(f"FATAL ERROR: Error occurred during execution of {exec_cmd}") @logit(logger) @@ -43,22 +46,19 @@ def link_executable(task_config: AttrDict, exe_name: str) -> None: @logit(logger) def prep_input_nml(task_config: AttrDict) -> None: - """Prepare the input.nml file - TODO: Use jinja2 instead of f90nml + """Prepare the mom_input.nml file """ - # stage input.nml - mom_input_nml_tmpl_src = os.path.join(task_config.HOMEgdas, 'parm', 'soca', 'fms', 'input.nml') + # stage input.nml.j2 + mom_input_nml_tmpl_src = os.path.join(task_config.PARMsoca, 'fms', 'input.nml.j2') mom_input_nml_tmpl = os.path.join(task_config.DATA, 'mom_input.nml.tmpl') FileHandler({'copy': [[mom_input_nml_tmpl_src, mom_input_nml_tmpl]]}).sync() # swap date and stacksize - domain_stack_size = task_config.DOMAIN_STACK_SIZE - ymdhms = [int(s) for s in task_config.MARINE_WINDOW_END.strftime('%Y,%m,%d,%H,%M,%S').split(',')] - with open(mom_input_nml_tmpl, 'r') as nml_file: - nml = f90nml.read(nml_file) - nml['ocean_solo_nml']['date_init'] = ymdhms - nml['fms_nml']['domains_stack_size'] = int(domain_stack_size) - nml.write('mom_input.nml') + date_init = [int(s) for s in task_config.MARINE_WINDOW_END.strftime('%Y,%m,%d,%H,%M,%S').split(',')] + input_nml_config = {'domain_stack_size': task_config.DOMAIN_STACK_SIZE, + 'date_init': date_init} + jinja_input_nml = jinja.Jinja(mom_input_nml_tmpl, input_nml_config) + jinja_input_nml.save('mom_input.nml') @logit(logger) @@ -74,3 +74,95 @@ def stage_ens_mem(task_config: AttrDict) -> None: letkf_stage_list = parse_j2yaml(task_config.MARINE_ENSDA_STAGE_BKG_YAML_TMPL, ensbkgconf) logger.info(f"{letkf_stage_list}") FileHandler(letkf_stage_list).sync() + + +@logit(logger) +def test_hist_date(histfile: str, ref_date: datetime) -> None: + """ + Check that the date in the MOM6 history file is the expected one for the cycle. + TODO: Implement the same for seaice + """ + + ncf = Dataset(histfile, 'r') + hist_date = dparser.parse(ncf.variables['time'].units, fuzzy=True) + timedelta(hours=int(ncf.variables['time'][0])) + ncf.close() + logger.info(f"*** history file date: {hist_date} expected date: {ref_date}") + + if hist_date != ref_date: + raise ValueError(f"FATAL ERROR: Inconsistent bkg date'") + + +@logit(logger) +def gen_bkg_list(bkg_path: str, window_begin=' ', yaml_name='bkg.yaml', ice_rst=False) -> None: + """ + Generate a YAML of the list of backgrounds for the pseudo model + """ + + # Pseudo model parameters (time step, start date) + # TODO: make this a parameter + dt_pseudo = 3 + bkg_date = window_begin + + # Construct list of background file names + cyc = str(os.getenv('cyc')).zfill(2) + gcyc = str((int(cyc) - 6) % 24).zfill(2) # previous cycle + fcst_hrs = list(range(6, 10, dt_pseudo)) + files = [] + for fcst_hr in fcst_hrs: + files.append(os.path.join(bkg_path, f"ocean.bkg.f{str(fcst_hr).zfill(3)}.nc")) + + # Identify the ocean background that will be used for the vertical coordinate remapping + ocn_filename_ic = './INPUT/MOM.res.nc' + test_hist_date(ocn_filename_ic, bkg_date) # assert date of the history file is correct + + # Copy/process backgrounds and generate background yaml list + bkg_list = [] + for bkg in files: + logger.info(f"****************** bkg: {bkg}") + # assert validity of the ocean bkg date, remove basename + bkg_date = bkg_date + timedelta(hours=dt_pseudo) + test_hist_date(bkg, bkg_date) + ocn_filename = os.path.splitext(os.path.basename(bkg))[0] + '.nc' + + # prepare the seaice background, aggregate if the backgrounds are CICE restarts + ice_filename = ocn_filename.replace("ocean", "ice") + + # prepare list of ocean and ice bkg to be copied to RUNDIR + bkg_dict = {'date': bkg_date.strftime('%Y-%m-%dT%H:%M:%SZ'), + 'basename': './bkg/', + 'ocn_filename': ocn_filename, + 'ice_filename': ice_filename, + 'read_from_file': 1} + + bkg_list.append(bkg_dict) + + # save pseudo model yaml configuration + save_as_yaml(bkg_list, yaml_name) + + +@logit(logger) +def clean_empty_obsspaces(config, target, app='var'): + """ + Remove obs spaces that point to non-existent file and save + """ + + # obs space dictionary depth is dependent on the application + if app == 'var': + obs_spaces = config['cost function']['observations']['observers'] + else: + raise ValueError(f"FATAL ERROR: obs space cleaning not implemented for {app}") + + # remove obs spaces that point to a non existant file + cleaned_obs_spaces = [] + for obs_space in obs_spaces: + fname = obs_space['obs space']['obsdatain']['engine']['obsfile'] + if os.path.isfile(fname): + cleaned_obs_spaces.append(obs_space) + else: + logger.info(f"WARNING: {fname} does not exist, removing obs space") + + # update obs spaces + config['cost function']['observations']['observers'] = cleaned_obs_spaces + + # save cleaned yaml + save_as_yaml(config, target) diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index 4bb473f454..19f4dd607b 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -44,10 +44,10 @@ def _get_app_configs(self): configs += ['anal', 'analdiag'] if self.do_jediocnvar: - configs += ['prepoceanobs', 'ocnanalprep', 'marinebmat', 'ocnanalrun'] + configs += ['prepoceanobs', 'marineanlinit', 'marinebmat', 'marineanlvar'] if self.do_hybvar: configs += ['ocnanalecen'] - configs += ['ocnanalchkpt', 'ocnanalpost'] + configs += ['marineanlchkpt', 'marineanlfinal'] if self.do_vrfy_oceanda: configs += ['ocnanalvrfy'] @@ -146,10 +146,10 @@ def get_task_names(self): gdas_gfs_common_tasks_before_fcst += ['anal'] if self.do_jediocnvar: - gdas_gfs_common_tasks_before_fcst += ['prepoceanobs', 'ocnanalprep', 'marinebmat', 'ocnanalrun'] + gdas_gfs_common_tasks_before_fcst += ['prepoceanobs', 'marineanlinit', 'marinebmat', 'marineanlvar'] if self.do_hybvar: gdas_gfs_common_tasks_before_fcst += ['ocnanalecen'] - gdas_gfs_common_tasks_before_fcst += ['ocnanalchkpt', 'ocnanalpost'] + gdas_gfs_common_tasks_before_fcst += ['marineanlchkpt', 'marineanlfinal'] if self.do_vrfy_oceanda: gdas_gfs_common_tasks_before_fcst += ['ocnanalvrfy'] diff --git a/workflow/hosts/awspw.yaml b/workflow/hosts/awspw.yaml index a9c708253e..ef17d8f2f4 100644 --- a/workflow/hosts/awspw.yaml +++ b/workflow/hosts/awspw.yaml @@ -18,6 +18,7 @@ CHGRP_RSTPROD: 'YES' CHGRP_CMD: 'chgrp rstprod' # TODO: This is not yet supported. HPSSARCH: 'NO' HPSS_PROJECT: emc-global #TODO: See `ATARDIR` below. +BASE_DATA: '/bucket/global-workflow-shared-data' BASE_IC: '/bucket/global-workflow-shared-data/ICSDIR' LOCALARCH: 'NO' ATARDIR: '' # TODO: This will not yet work from AWS. diff --git a/workflow/hosts/azurepw.yaml b/workflow/hosts/azurepw.yaml index d59736e653..1769f9ee19 100644 --- a/workflow/hosts/azurepw.yaml +++ b/workflow/hosts/azurepw.yaml @@ -18,6 +18,7 @@ CHGRP_RSTPROD: 'YES' CHGRP_CMD: 'chgrp rstprod' # TODO: This is not yet supported. HPSSARCH: 'NO' HPSS_PROJECT: emc-global #TODO: See `ATARDIR` below. +BASE_DATA: '/bucket/global-workflow-shared-data' BASE_IC: '/bucket/global-workflow-shared-data/ICSDIR' LOCALARCH: 'NO' ATARDIR: '' # TODO: This will not yet work from AZURE. diff --git a/workflow/hosts/gaea.yaml b/workflow/hosts/gaea.yaml index 9297fed24a..5a37b5dabf 100644 --- a/workflow/hosts/gaea.yaml +++ b/workflow/hosts/gaea.yaml @@ -1,5 +1,6 @@ BASE_GIT: '/gpfs/f5/ufs-ard/world-shared/global/glopara/data/git' DMPDIR: '/gpfs/f5/ufs-ard/world-shared/global/glopara/data/dump' +BASE_DATA: '/gpfs/f5/ufs-ard/world-shared/global/glopara/data' BASE_IC: '/gpfs/f5/ufs-ard/world-shared/global/glopara/data/ICSDIR' PACKAGEROOT: '/gpfs/f5/ufs-ard/world-shared/global/glopara/data/nwpara' COMROOT: '/gpfs/f5/ufs-ard/world-shared/global/glopara/data/com' diff --git a/workflow/hosts/googlepw.yaml b/workflow/hosts/googlepw.yaml index 2bd9439d5f..daf6cd1eb2 100644 --- a/workflow/hosts/googlepw.yaml +++ b/workflow/hosts/googlepw.yaml @@ -18,6 +18,7 @@ CHGRP_RSTPROD: 'YES' CHGRP_CMD: 'chgrp rstprod' # TODO: This is not yet supported. HPSSARCH: 'NO' HPSS_PROJECT: emc-global #TODO: See `ATARDIR` below. +BASE_DATA: '/bucket/global-workflow-shared-data' BASE_IC: '/bucket/global-workflow-shared-data/ICSDIR' LOCALARCH: 'NO' ATARDIR: '' # TODO: This will not yet work from GOOGLE. diff --git a/workflow/hosts/hera.yaml b/workflow/hosts/hera.yaml index 4ace199470..fa2c351aa1 100644 --- a/workflow/hosts/hera.yaml +++ b/workflow/hosts/hera.yaml @@ -1,5 +1,6 @@ BASE_GIT: '/scratch1/NCEPDEV/global/glopara/git' DMPDIR: '/scratch1/NCEPDEV/global/glopara/dump' +BASE_DATA: '/scratch1/NCEPDEV/global/glopara/data' BASE_IC: '/scratch1/NCEPDEV/global/glopara/data/ICSDIR' PACKAGEROOT: '/scratch1/NCEPDEV/global/glopara/nwpara' COMINsyn: '/scratch1/NCEPDEV/global/glopara/com/gfs/prod/syndat' diff --git a/workflow/hosts/hercules.yaml b/workflow/hosts/hercules.yaml index 9d6339a48e..73fde6cde6 100644 --- a/workflow/hosts/hercules.yaml +++ b/workflow/hosts/hercules.yaml @@ -1,5 +1,6 @@ BASE_GIT: '/work/noaa/global/glopara/git_rocky9' DMPDIR: '/work/noaa/rstprod/dump' +BASE_DATA: '/work/noaa/global/glopara/data' BASE_IC: '/work/noaa/global/glopara/data/ICSDIR' PACKAGEROOT: '/work/noaa/global/glopara/nwpara' COMINsyn: '/work/noaa/global/glopara/com/gfs/prod/syndat' diff --git a/workflow/hosts/jet.yaml b/workflow/hosts/jet.yaml index 21e815c9b2..80957083e0 100644 --- a/workflow/hosts/jet.yaml +++ b/workflow/hosts/jet.yaml @@ -1,5 +1,6 @@ BASE_GIT: '/lfs4/HFIP/hfv3gfs/glopara/git' DMPDIR: '/lfs4/HFIP/hfv3gfs/glopara/dump' +BASE_DATA: '/lfs5/HFIP/hfv3gfs/glopara/data' BASE_IC: '/mnt/lfs4/HFIP/hfv3gfs/glopara/data/ICSDIR' PACKAGEROOT: '/lfs4/HFIP/hfv3gfs/glopara/nwpara' COMINsyn: '/lfs4/HFIP/hfv3gfs/glopara/com/gfs/prod/syndat' diff --git a/workflow/hosts/orion.yaml b/workflow/hosts/orion.yaml index 4ec78fc8cc..d47b2b2bab 100644 --- a/workflow/hosts/orion.yaml +++ b/workflow/hosts/orion.yaml @@ -1,5 +1,6 @@ BASE_GIT: '/work/noaa/global/glopara/git_rocky9' DMPDIR: '/work/noaa/rstprod/dump' +BASE_DATA: '/work/noaa/global/glopara/data' BASE_IC: '/work/noaa/global/glopara/data/ICSDIR' PACKAGEROOT: '/work/noaa/global/glopara/nwpara' COMINsyn: '/work/noaa/global/glopara/com/gfs/prod/syndat' diff --git a/workflow/hosts/s4.yaml b/workflow/hosts/s4.yaml index c2af9728f2..b93fefec39 100644 --- a/workflow/hosts/s4.yaml +++ b/workflow/hosts/s4.yaml @@ -1,5 +1,6 @@ BASE_GIT: '/data/prod/glopara/git' DMPDIR: '/data/prod/glopara/dump' +BASE_DATA: '/data/prod/glopara' BASE_IC: '/data/prod/glopara/coupled_ICs' PACKAGEROOT: '/data/prod/glopara/nwpara' COMINsyn: '/data/prod/glopara/com/gfs/prod/syndat' diff --git a/workflow/hosts/wcoss2.yaml b/workflow/hosts/wcoss2.yaml index bf2cc41c45..15f0705f91 100644 --- a/workflow/hosts/wcoss2.yaml +++ b/workflow/hosts/wcoss2.yaml @@ -1,5 +1,6 @@ BASE_GIT: '/lfs/h2/emc/global/save/emc.global/git' DMPDIR: '/lfs/h2/emc/dump/noscrub/dump' +BASE_DATA: '/lfs/h2/emc/global/noscrub/emc.global/data' BASE_IC: '/lfs/h2/emc/global/noscrub/emc.global/data/ICSDIR' PACKAGEROOT: '${PACKAGEROOT:-"/lfs/h1/ops/prod/packages"}' COMINsyn: '/lfs/h1/ops/prod/com/gfs/v16.3/syndat' diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 89da933d00..ab972ad08b 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -692,7 +692,7 @@ def marinebmat(self): return task - def ocnanalprep(self): + def marineanlinit(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.run}prepoceanobs'} @@ -703,14 +703,14 @@ def ocnanalprep(self): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - resources = self.get_resource('ocnanalprep') - task_name = f'{self.run}ocnanalprep' + resources = self.get_resource('marineanlinit') + task_name = f'{self.run}marineanlinit' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/ocnanalprep.sh', + 'command': f'{self.HOMEgfs}/jobs/rocoto/marineanlinit.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -720,21 +720,21 @@ def ocnanalprep(self): return task - def ocnanalrun(self): + def marineanlvar(self): deps = [] - dep_dict = {'type': 'task', 'name': f'{self.run}ocnanalprep'} + dep_dict = {'type': 'task', 'name': f'{self.run}marineanlinit'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) - resources = self.get_resource('ocnanalrun') - task_name = f'{self.run}ocnanalrun' + resources = self.get_resource('marineanlvar') + task_name = f'{self.run}marineanlvar' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/ocnanalrun.sh', + 'command': f'{self.HOMEgfs}/jobs/rocoto/marineanlvar.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -747,7 +747,7 @@ def ocnanalrun(self): def ocnanalecen(self): deps = [] - dep_dict = {'type': 'task', 'name': f'{self.run}ocnanalrun'} + dep_dict = {'type': 'task', 'name': f'{self.run}marineanlvar'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) @@ -768,13 +768,13 @@ def ocnanalecen(self): return task - def ocnanalchkpt(self): + def marineanlchkpt(self): deps = [] if self.app_config.do_hybvar: dep_dict = {'type': 'task', 'name': f'{self.run}ocnanalecen'} else: - dep_dict = {'type': 'task', 'name': f'{self.run}ocnanalrun'} + dep_dict = {'type': 'task', 'name': f'{self.run}marineanlvar'} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_mergensst: data = f'&ROTDIR;/{self.run}.@Y@m@d/@H/atmos/{self.run}.t@Hz.sfcanl.nc' @@ -782,14 +782,14 @@ def ocnanalchkpt(self): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - resources = self.get_resource('ocnanalchkpt') - task_name = f'{self.run}ocnanalchkpt' + resources = self.get_resource('marineanlchkpt') + task_name = f'{self.run}marineanlchkpt' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/ocnanalchkpt.sh', + 'command': f'{self.HOMEgfs}/jobs/rocoto/marineanlchkpt.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -799,21 +799,21 @@ def ocnanalchkpt(self): return task - def ocnanalpost(self): + def marineanlfinal(self): deps = [] - dep_dict = {'type': 'task', 'name': f'{self.run}ocnanalchkpt'} + dep_dict = {'type': 'task', 'name': f'{self.run}marineanlchkpt'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - resources = self.get_resource('ocnanalpost') - task_name = f'{self.run}ocnanalpost' + resources = self.get_resource('marineanlfinal') + task_name = f'{self.run}marineanlfinal' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/ocnanalpost.sh', + 'command': f'{self.HOMEgfs}/jobs/rocoto/marineanlfinal.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -826,7 +826,7 @@ def ocnanalpost(self): def ocnanalvrfy(self): deps = [] - dep_dict = {'type': 'task', 'name': f'{self.run}ocnanalpost'} + dep_dict = {'type': 'task', 'name': f'{self.run}marineanlfinal'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -932,7 +932,7 @@ def _fcst_cycled(self): dependencies = rocoto.create_dependency(dep=dep) if self.app_config.do_jediocnvar: - dep_dict = {'type': 'task', 'name': f'{self.run}ocnanalpost'} + dep_dict = {'type': 'task', 'name': f'{self.run}marineanlfinal'} dependencies.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_aero and self.run in self.app_config.aero_anl_runs: diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index df2b0467db..8a32827377 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -15,7 +15,7 @@ class Tasks: 'prep', 'anal', 'sfcanl', 'analcalc', 'analdiag', 'arch', "cleanup", 'prepatmiodaobs', 'atmanlinit', 'atmanlvar', 'atmanlfv3inc', 'atmanlfinal', 'prepoceanobs', - 'ocnanalprep', 'marinebmat', 'ocnanalrun', 'ocnanalecen', 'ocnanalchkpt', 'ocnanalpost', 'ocnanalvrfy', + 'marineanlinit', 'marinebmat', 'marineanlvar', 'ocnanalecen', 'marineanlchkpt', 'marineanlfinal', 'ocnanalvrfy', 'earc', 'ecen', 'echgres', 'ediag', 'efcs', 'eobs', 'eomg', 'epos', 'esfc', 'eupd', 'atmensanlinit', 'atmensanlobs', 'atmensanlsol', 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal', From a443fd183ab3b802832d5d0bc5082e84b30951d0 Mon Sep 17 00:00:00 2001 From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> Date: Fri, 27 Sep 2024 00:23:48 -0400 Subject: [PATCH 2/2] Enable parallel metp jobs and fix race condition with the gfscleanup job (#2907) This brings in a change to EMC_verif-global that offsets the start time of parallel instances when running metp jobs. This prevents Python from attempting to create the same directory in multiple instances. Simultaneously, this also fixes an issue with the `gfscleanup` job potentially running before the `metp` jobs. Resolves #2906 Resolves #2899 --- parm/archive/enkf.yaml.j2 | 2 +- parm/config/gfs/config.resources | 4 ++-- sorc/verif-global.fd | 2 +- workflow/rocoto/gfs_tasks.py | 8 ++++++-- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/parm/archive/enkf.yaml.j2 b/parm/archive/enkf.yaml.j2 index a95046d4d6..89fd44500b 100644 --- a/parm/archive/enkf.yaml.j2 +++ b/parm/archive/enkf.yaml.j2 @@ -27,7 +27,7 @@ enkf: {% else %} {% set steps = ["eobs", "eupd"] %} {% for mem in range(1, nmem_ens + 1) %} - {% do steps.append("eomg_mem{{ '%03d' % mem }}") %} + {% do steps.append("eomg_mem" ~ '%03d' % mem) %} {% endfor %} {% endif %} {% endif %} diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index d512f1f885..5024bc8873 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -988,8 +988,8 @@ case ${step} in threads_per_task=1 walltime_gdas="03:00:00" walltime_gfs="06:00:00" - ntasks=1 - tasks_per_node=1 + ntasks=4 + tasks_per_node=4 export memory="80G" ;; diff --git a/sorc/verif-global.fd b/sorc/verif-global.fd index 92904d2c43..e7e6bc4358 160000 --- a/sorc/verif-global.fd +++ b/sorc/verif-global.fd @@ -1 +1 @@ -Subproject commit 92904d2c431969345968f74e676717057ec0042a +Subproject commit e7e6bc43584e0b8911819b8f875cc8ee747db76d diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index ab972ad08b..6b9d6358c6 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -855,8 +855,8 @@ def fcst(self): try: task = fcst_map[self.app_config.mode]() except KeyError: - raise NotImplementedError(f'{self.app_config.mode} is not a valid type.\n' + - 'Currently supported forecast types are:\n' + + raise NotImplementedError(f'{self.app_config.mode} is not a valid type.\n' + f'Currently supported forecast types are:\n' f'{" | ".join(fcst_map.keys())}') return task @@ -2330,6 +2330,10 @@ def cleanup(self): dep_dict = {'type': 'task', 'name': f'{self.run}npoess_pgrb2_0p5deg'} deps.append(rocoto.add_dependency(dep_dict)) + if self.app_config.do_metp and self.run in ['gfs']: + dep_dict = {'type': 'metatask', 'name': f'{self.run}metp'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('cleanup')